2023-10-09T05:38:25.100 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T05:38:25.101 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --name teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516 --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/orig.config.yaml 2023-10-09T05:38:25.167 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T05:38:25.259 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7417516' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 kernel: hwe: true workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 3687 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi125.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi155.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T05:38:25.260 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T05:38:25.261 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T05:38:25.261 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T05:38:25.262 INFO:teuthology.task.internal:Checking packages... 2023-10-09T05:38:25.285 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T05:38:25.285 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:38:25.285 INFO:teuthology.packaging:ref: None 2023-10-09T05:38:25.285 INFO:teuthology.packaging:tag: None 2023-10-09T05:38:25.286 INFO:teuthology.packaging:branch: reef 2023-10-09T05:38:25.286 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:38:25.286 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-09T05:38:25.737 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610-g397de1b0-1focal 2023-10-09T05:38:25.737 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T05:38:25.787 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T05:38:25.787 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T05:38:25.814 INFO:teuthology.task.internal:Saving configuration 2023-10-09T05:38:25.839 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T05:38:25.854 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:38:25.883 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi125.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:32:10.446807', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T05:38:25.909 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:32:10.448279', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-09T05:38:25.909 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T05:38:25.924 INFO:teuthology.task.internal:roles: ubuntu@smithi125.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T05:38:25.924 INFO:teuthology.task.internal:roles: ubuntu@smithi155.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T05:38:25.925 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T05:38:26.012 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f90ee82e160>, signals=[15]) 2023-10-09T05:38:26.012 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T05:38:26.026 INFO:teuthology.task.internal:Opening connections... 2023-10-09T05:38:26.026 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T05:38:26.028 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:38:26.077 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi155.front.sepia.ceph.com 2023-10-09T05:38:26.078 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:38:26.127 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T05:38:26.137 DEBUG:teuthology.orchestra.run.smithi125:> uname -m 2023-10-09T05:38:26.161 INFO:teuthology.orchestra.run.smithi125.stdout:x86_64 2023-10-09T05:38:26.162 DEBUG:teuthology.orchestra.run.smithi125:> cat /etc/os-release 2023-10-09T05:38:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:NAME="Ubuntu" 2023-10-09T05:38:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T05:38:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:ID=ubuntu 2023-10-09T05:38:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:ID_LIKE=debian 2023-10-09T05:38:26.169 INFO:teuthology.orchestra.run.smithi125.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION_ID="20.04" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION_CODENAME=focal 2023-10-09T05:38:26.170 INFO:teuthology.orchestra.run.smithi125.stdout:UBUNTU_CODENAME=focal 2023-10-09T05:38:26.171 INFO:teuthology.lock.ops:Updating smithi125.front.sepia.ceph.com on lock server 2023-10-09T05:38:26.193 DEBUG:teuthology.orchestra.run.smithi155:> uname -m 2023-10-09T05:38:26.205 INFO:teuthology.orchestra.run.smithi155.stdout:x86_64 2023-10-09T05:38:26.206 DEBUG:teuthology.orchestra.run.smithi155:> cat /etc/os-release 2023-10-09T05:38:26.254 INFO:teuthology.orchestra.run.smithi155.stdout:NAME="Ubuntu" 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:ID=ubuntu 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:ID_LIKE=debian 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION_ID="20.04" 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-09T05:38:26.255 INFO:teuthology.orchestra.run.smithi155.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-09T05:38:26.256 INFO:teuthology.orchestra.run.smithi155.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-09T05:38:26.256 INFO:teuthology.orchestra.run.smithi155.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-09T05:38:26.256 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION_CODENAME=focal 2023-10-09T05:38:26.256 INFO:teuthology.orchestra.run.smithi155.stdout:UBUNTU_CODENAME=focal 2023-10-09T05:38:26.256 INFO:teuthology.lock.ops:Updating smithi155.front.sepia.ceph.com on lock server 2023-10-09T05:38:26.332 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T05:38:26.343 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T05:38:26.352 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T05:38:26.353 DEBUG:teuthology.orchestra.run.smithi125:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:38:26.355 DEBUG:teuthology.orchestra.run.smithi155:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:38:26.361 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T05:38:26.371 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T05:38:26.371 DEBUG:teuthology.orchestra.run.smithi125:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:38:26.402 DEBUG:teuthology.orchestra.run.smithi155:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:38:26.424 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T05:38:26.482 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T05:38:26.499 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T05:38:26.500 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-09T05:38:26.500 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-09T05:38:26.500 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-09T05:38:26.500 DEBUG:teuthology.orchestra.run.smithi125:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:38:26.501 DEBUG:teuthology.orchestra.run.smithi155:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:38:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.506 DEBUG:teuthology.orchestra.run.smithi125:> uname -r 2023-10-09T05:38:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:38:26.507 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2023-10-09T05:38:26.513 INFO:teuthology.orchestra.run.smithi125.stdout:5.4.0-124-generic 2023-10-09T05:38:26.514 INFO:teuthology.task.kernel:Running kernel on smithi125: 5.4.0-124-generic 2023-10-09T05:38:26.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-get clean 2023-10-09T05:38:26.515 INFO:teuthology.orchestra.run.smithi155.stdout:5.4.0-124-generic 2023-10-09T05:38:26.515 INFO:teuthology.task.kernel:Running kernel on smithi155: 5.4.0-124-generic 2023-10-09T05:38:26.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-get clean 2023-10-09T05:38:26.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-get update 2023-10-09T05:38:26.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-get update 2023-10-09T05:38:26.816 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:38:26.817 INFO:teuthology.orchestra.run.smithi125.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T05:38:26.817 INFO:teuthology.orchestra.run.smithi125.stdout:Get:3 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T05:38:26.874 INFO:teuthology.orchestra.run.smithi125.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T05:38:27.043 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:38:27.053 INFO:teuthology.orchestra.run.smithi155.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-09T05:38:27.128 INFO:teuthology.orchestra.run.smithi155.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-09T05:38:27.152 INFO:teuthology.orchestra.run.smithi155.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-09T05:38:27.209 INFO:teuthology.orchestra.run.smithi125.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T05:38:27.285 INFO:teuthology.orchestra.run.smithi125.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T05:38:27.330 INFO:teuthology.orchestra.run.smithi125.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T05:38:27.331 INFO:teuthology.orchestra.run.smithi125.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T05:38:27.331 INFO:teuthology.orchestra.run.smithi125.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T05:38:27.342 INFO:teuthology.orchestra.run.smithi125.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T05:38:27.342 INFO:teuthology.orchestra.run.smithi125.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T05:38:27.344 INFO:teuthology.orchestra.run.smithi125.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T05:38:27.346 INFO:teuthology.orchestra.run.smithi125.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T05:38:27.351 INFO:teuthology.orchestra.run.smithi125.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T05:38:27.354 INFO:teuthology.orchestra.run.smithi125.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T05:38:27.359 INFO:teuthology.orchestra.run.smithi125.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T05:38:27.361 INFO:teuthology.orchestra.run.smithi125.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T05:38:27.364 INFO:teuthology.orchestra.run.smithi125.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T05:38:27.365 INFO:teuthology.orchestra.run.smithi125.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T05:38:27.365 INFO:teuthology.orchestra.run.smithi125.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T05:38:27.365 INFO:teuthology.orchestra.run.smithi125.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T05:38:27.431 INFO:teuthology.orchestra.run.smithi125.stdout:Get:22 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T05:38:27.444 INFO:teuthology.orchestra.run.smithi155.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [893 kB] 2023-10-09T05:38:27.466 INFO:teuthology.orchestra.run.smithi125.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T05:38:27.477 INFO:teuthology.orchestra.run.smithi125.stdout:Get:24 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T05:38:27.485 INFO:teuthology.orchestra.run.smithi125.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T05:38:27.486 INFO:teuthology.orchestra.run.smithi125.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T05:38:27.486 INFO:teuthology.orchestra.run.smithi125.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T05:38:27.486 INFO:teuthology.orchestra.run.smithi125.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T05:38:27.486 INFO:teuthology.orchestra.run.smithi125.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T05:38:27.489 INFO:teuthology.orchestra.run.smithi125.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T05:38:27.489 INFO:teuthology.orchestra.run.smithi125.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T05:38:27.490 INFO:teuthology.orchestra.run.smithi125.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T05:38:27.513 INFO:teuthology.orchestra.run.smithi125.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T05:38:27.516 INFO:teuthology.orchestra.run.smithi125.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T05:38:27.516 INFO:teuthology.orchestra.run.smithi125.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T05:38:27.523 INFO:teuthology.orchestra.run.smithi125.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T05:38:27.529 INFO:teuthology.orchestra.run.smithi125.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T05:38:27.530 INFO:teuthology.orchestra.run.smithi125.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T05:38:27.530 INFO:teuthology.orchestra.run.smithi125.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T05:38:27.531 INFO:teuthology.orchestra.run.smithi125.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T05:38:27.531 INFO:teuthology.orchestra.run.smithi125.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T05:38:27.550 INFO:teuthology.orchestra.run.smithi155.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,890 kB] 2023-10-09T05:38:27.550 INFO:teuthology.orchestra.run.smithi125.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T05:38:27.600 INFO:teuthology.orchestra.run.smithi155.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [473 kB] 2023-10-09T05:38:27.603 INFO:teuthology.orchestra.run.smithi155.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-09T05:38:27.604 INFO:teuthology.orchestra.run.smithi155.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-09T05:38:27.604 INFO:teuthology.orchestra.run.smithi155.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,383 kB] 2023-10-09T05:38:27.625 INFO:teuthology.orchestra.run.smithi155.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [334 kB] 2023-10-09T05:38:27.627 INFO:teuthology.orchestra.run.smithi155.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T05:38:27.628 INFO:teuthology.orchestra.run.smithi155.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,121 kB] 2023-10-09T05:38:27.637 INFO:teuthology.orchestra.run.smithi155.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [749 kB] 2023-10-09T05:38:27.643 INFO:teuthology.orchestra.run.smithi155.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [268 kB] 2023-10-09T05:38:27.645 INFO:teuthology.orchestra.run.smithi155.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-09T05:38:27.646 INFO:teuthology.orchestra.run.smithi155.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-09T05:38:27.647 INFO:teuthology.orchestra.run.smithi155.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-09T05:38:27.647 INFO:teuthology.orchestra.run.smithi155.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-09T05:38:27.647 INFO:teuthology.orchestra.run.smithi155.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-09T05:38:27.669 INFO:teuthology.orchestra.run.smithi155.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-09T05:38:27.669 INFO:teuthology.orchestra.run.smithi155.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-09T05:38:27.669 INFO:teuthology.orchestra.run.smithi155.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-09T05:38:27.670 INFO:teuthology.orchestra.run.smithi155.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-09T05:38:27.670 INFO:teuthology.orchestra.run.smithi155.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-09T05:38:27.670 INFO:teuthology.orchestra.run.smithi155.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-09T05:38:27.750 INFO:teuthology.orchestra.run.smithi155.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,500 kB] 2023-10-09T05:38:28.225 INFO:teuthology.orchestra.run.smithi155.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [660 kB] 2023-10-09T05:38:28.239 INFO:teuthology.orchestra.run.smithi155.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [390 kB] 2023-10-09T05:38:28.248 INFO:teuthology.orchestra.run.smithi155.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-09T05:38:28.248 INFO:teuthology.orchestra.run.smithi155.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-09T05:38:28.249 INFO:teuthology.orchestra.run.smithi155.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,267 kB] 2023-10-09T05:38:28.324 INFO:teuthology.orchestra.run.smithi155.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [317 kB] 2023-10-09T05:38:28.331 INFO:teuthology.orchestra.run.smithi155.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-09T05:38:28.331 INFO:teuthology.orchestra.run.smithi155.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [616 kB] 2023-10-09T05:38:28.345 INFO:teuthology.orchestra.run.smithi155.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [890 kB] 2023-10-09T05:38:28.524 INFO:teuthology.orchestra.run.smithi155.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [186 kB] 2023-10-09T05:38:28.938 INFO:teuthology.orchestra.run.smithi155.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-09T05:38:28.944 INFO:teuthology.orchestra.run.smithi155.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-09T05:38:28.951 INFO:teuthology.orchestra.run.smithi155.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-09T05:38:28.954 INFO:teuthology.orchestra.run.smithi155.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-09T05:38:28.956 INFO:teuthology.orchestra.run.smithi155.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-09T05:38:31.079 INFO:teuthology.orchestra.run.smithi125.stdout:Fetched 17.6 MB in 3s (7,025 kB/s) 2023-10-09T05:38:31.520 INFO:teuthology.orchestra.run.smithi155.stdout:Fetched 17.6 MB in 3s (6,376 kB/s) 2023-10-09T05:38:32.510 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:38:32.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T05:38:32.608 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:38:32.805 INFO:teuthology.orchestra.run.smithi125.stdout:Building dependency tree... 2023-10-09T05:38:32.805 INFO:teuthology.orchestra.run.smithi125.stdout:Reading state information... 2023-10-09T05:38:32.890 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:38:32.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T05:38:32.969 INFO:teuthology.orchestra.run.smithi125.stdout:The following additional packages will be installed: 2023-10-09T05:38:32.969 INFO:teuthology.orchestra.run.smithi125.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T05:38:32.970 INFO:teuthology.orchestra.run.smithi125.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T05:38:32.970 INFO:teuthology.orchestra.run.smithi125.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T05:38:32.971 INFO:teuthology.orchestra.run.smithi125.stdout:Suggested packages: 2023-10-09T05:38:32.971 INFO:teuthology.orchestra.run.smithi125.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T05:38:32.971 INFO:teuthology.orchestra.run.smithi125.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T05:38:32.984 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:38:33.026 INFO:teuthology.orchestra.run.smithi125.stdout:The following NEW packages will be installed: 2023-10-09T05:38:33.027 INFO:teuthology.orchestra.run.smithi125.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T05:38:33.027 INFO:teuthology.orchestra.run.smithi125.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T05:38:33.027 INFO:teuthology.orchestra.run.smithi125.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T05:38:33.027 INFO:teuthology.orchestra.run.smithi125.stdout: thermald upower usbmuxd 2023-10-09T05:38:33.184 INFO:teuthology.orchestra.run.smithi155.stdout:Building dependency tree... 2023-10-09T05:38:33.185 INFO:teuthology.orchestra.run.smithi155.stdout:Reading state information... 2023-10-09T05:38:33.231 INFO:teuthology.orchestra.run.smithi125.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T05:38:33.231 INFO:teuthology.orchestra.run.smithi125.stdout:Need to get 90.9 MB of archives. 2023-10-09T05:38:33.231 INFO:teuthology.orchestra.run.smithi125.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T05:38:33.231 INFO:teuthology.orchestra.run.smithi125.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T05:38:33.355 INFO:teuthology.orchestra.run.smithi155.stdout:The following additional packages will be installed: 2023-10-09T05:38:33.355 INFO:teuthology.orchestra.run.smithi155.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T05:38:33.355 INFO:teuthology.orchestra.run.smithi155.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-modules-5.15.0-86-generic 2023-10-09T05:38:33.356 INFO:teuthology.orchestra.run.smithi155.stdout: linux-modules-extra-5.15.0-86-generic thermald upower usbmuxd 2023-10-09T05:38:33.356 INFO:teuthology.orchestra.run.smithi155.stdout:Suggested packages: 2023-10-09T05:38:33.357 INFO:teuthology.orchestra.run.smithi155.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-09T05:38:33.357 INFO:teuthology.orchestra.run.smithi155.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-86-generic 2023-10-09T05:38:33.411 INFO:teuthology.orchestra.run.smithi155.stdout:The following NEW packages will be installed: 2023-10-09T05:38:33.412 INFO:teuthology.orchestra.run.smithi155.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-09T05:38:33.412 INFO:teuthology.orchestra.run.smithi155.stdout: libusbmuxd6 linux-image-5.15.0-86-generic linux-image-generic-hwe-20.04 2023-10-09T05:38:33.412 INFO:teuthology.orchestra.run.smithi155.stdout: linux-modules-5.15.0-86-generic linux-modules-extra-5.15.0-86-generic 2023-10-09T05:38:33.412 INFO:teuthology.orchestra.run.smithi155.stdout: thermald upower usbmuxd 2023-10-09T05:38:33.464 INFO:teuthology.orchestra.run.smithi155.stdout:0 upgraded, 13 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T05:38:33.464 INFO:teuthology.orchestra.run.smithi155.stdout:Need to get 90.9 MB of archives. 2023-10-09T05:38:33.464 INFO:teuthology.orchestra.run.smithi155.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-09T05:38:33.464 INFO:teuthology.orchestra.run.smithi155.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-09T05:38:33.502 INFO:teuthology.orchestra.run.smithi125.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T05:38:33.521 INFO:teuthology.orchestra.run.smithi155.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-09T05:38:33.527 INFO:teuthology.orchestra.run.smithi155.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T05:38:33.530 INFO:teuthology.orchestra.run.smithi125.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-09T05:38:33.531 INFO:teuthology.orchestra.run.smithi155.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T05:38:33.545 INFO:teuthology.orchestra.run.smithi155.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T05:38:33.548 INFO:teuthology.orchestra.run.smithi125.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-09T05:38:33.553 INFO:teuthology.orchestra.run.smithi155.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T05:38:33.622 INFO:teuthology.orchestra.run.smithi125.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-09T05:38:33.658 INFO:teuthology.orchestra.run.smithi125.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [20.9 MB] 2023-10-09T05:38:33.875 INFO:teuthology.orchestra.run.smithi155.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T05:38:33.975 INFO:teuthology.orchestra.run.smithi155.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T05:38:34.593 INFO:teuthology.orchestra.run.smithi125.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [11.4 MB] 2023-10-09T05:38:34.649 INFO:teuthology.orchestra.run.smithi155.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T05:38:34.649 INFO:teuthology.orchestra.run.smithi155.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T05:38:34.709 INFO:teuthology.orchestra.run.smithi155.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T05:38:34.714 INFO:teuthology.orchestra.run.smithi155.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T05:38:34.717 INFO:teuthology.orchestra.run.smithi155.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T05:38:34.883 INFO:teuthology.orchestra.run.smithi125.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-86-generic amd64 5.15.0-86.96~20.04.1 [58.0 MB] 2023-10-09T05:38:35.380 INFO:teuthology.orchestra.run.smithi155.stdout:Fetched 90.9 MB in 1s (70.0 MB/s) 2023-10-09T05:38:35.540 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T05:38:36.849 INFO:teuthology.orchestra.run.smithi125.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.86.96~20.04.44 [2,648 B] 2023-10-09T05:38:36.850 INFO:teuthology.orchestra.run.smithi125.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-09T05:38:36.851 INFO:teuthology.orchestra.run.smithi125.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-09T05:38:37.021 INFO:teuthology.orchestra.run.smithi125.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-09T05:38:37.249 INFO:teuthology.orchestra.run.smithi155.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T05:38:37.254 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T05:38:37.365 INFO:teuthology.orchestra.run.smithi125.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-09T05:38:37.379 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T05:38:37.757 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T05:38:37.771 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T05:38:37.815 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T05:38:38.014 INFO:teuthology.orchestra.run.smithi125.stdout:Fetched 90.9 MB in 4s (20.9 MB/s) 2023-10-09T05:38:38.160 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T05:38:38.175 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T05:38:38.225 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-09T05:38:38.226 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T05:38:38.471 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T05:38:38.483 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T05:38:38.513 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T05:38:38.840 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T05:38:38.856 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T05:38:38.924 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T05:38:39.235 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T05:38:39.250 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:39.293 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:39.961 INFO:teuthology.orchestra.run.smithi125.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-09T05:38:39.966 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-09T05:38:40.080 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T05:38:40.450 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-09T05:38:40.465 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-09T05:38:40.508 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T05:38:40.836 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-09T05:38:40.850 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-09T05:38:40.886 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T05:38:41.214 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-09T05:38:41.226 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-09T05:38:41.289 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T05:38:41.584 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-09T05:38:41.597 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-09T05:38:41.634 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T05:38:41.910 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T05:38:41.925 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:41.953 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package linux-modules-5.15.0-86-generic. 2023-10-09T05:38:41.966 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../05-linux-modules-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:42.003 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:42.024 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:42.547 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T05:38:42.561 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:42.605 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:44.545 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package linux-image-5.15.0-86-generic. 2023-10-09T05:38:44.560 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../06-linux-image-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:44.670 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:45.181 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-86-generic. 2023-10-09T05:38:45.197 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-86-generic_5.15.0-86.96~20.04.1_amd64.deb ... 2023-10-09T05:38:45.239 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:49.430 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T05:38:49.445 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T05:38:49.488 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T05:38:49.824 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T05:38:49.838 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T05:38:49.882 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T05:38:50.252 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package thermald. 2023-10-09T05:38:50.266 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T05:38:50.318 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T05:38:50.848 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package upower. 2023-10-09T05:38:50.862 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T05:38:50.923 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T05:38:51.344 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T05:38:51.358 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T05:38:51.402 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T05:38:51.689 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T05:38:51.840 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T05:38:51.966 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T05:38:52.089 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-09T05:38:52.092 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T05:38:52.104 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.86.96~20.04.44_amd64.deb ... 2023-10-09T05:38:52.147 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T05:38:52.218 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T05:38:52.344 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T05:38:52.487 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T05:38:52.500 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-09T05:38:52.514 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-09T05:38:52.558 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T05:38:52.911 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package thermald. 2023-10-09T05:38:52.926 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-09T05:38:52.970 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T05:38:53.046 INFO:teuthology.orchestra.run.smithi155.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T05:38:53.099 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T05:38:53.296 INFO:teuthology.orchestra.run.smithi155.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T05:38:53.301 INFO:teuthology.orchestra.run.smithi155.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T05:38:53.302 INFO:teuthology.orchestra.run.smithi155.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T05:38:53.466 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package upower. 2023-10-09T05:38:53.481 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-09T05:38:53.548 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-09T05:38:53.794 INFO:teuthology.orchestra.run.smithi155.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T05:38:53.954 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package usbmuxd. 2023-10-09T05:38:53.971 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-09T05:38:54.009 INFO:teuthology.orchestra.run.smithi155.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T05:38:54.012 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T05:38:54.048 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T05:38:54.303 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-09T05:38:54.359 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T05:38:54.359 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T05:38:54.425 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-09T05:38:54.551 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-09T05:38:54.677 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-09T05:38:54.810 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:54.819 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-09T05:38:54.946 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-09T05:38:55.072 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-09T05:38:55.613 INFO:teuthology.orchestra.run.smithi125.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-09T05:38:55.667 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-09T05:38:55.827 INFO:teuthology.orchestra.run.smithi125.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-09T05:38:55.839 INFO:teuthology.orchestra.run.smithi125.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-09T05:38:55.839 INFO:teuthology.orchestra.run.smithi125.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-09T05:38:56.124 INFO:teuthology.orchestra.run.smithi155.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:38:56.124 INFO:teuthology.orchestra.run.smithi155.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T05:38:56.273 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:56.401 INFO:teuthology.orchestra.run.smithi125.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-09T05:38:56.624 INFO:teuthology.orchestra.run.smithi125.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-09T05:38:56.672 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-09T05:38:56.985 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T05:38:57.175 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-09T05:38:57.451 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:57.691 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:58.754 INFO:teuthology.orchestra.run.smithi125.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:38:58.754 INFO:teuthology.orchestra.run.smithi125.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-86-generic 2023-10-09T05:38:58.914 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up linux-modules-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:38:59.059 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T05:38:59.185 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T05:38:59.432 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T05:39:00.323 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up linux-modules-extra-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:39:00.838 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T05:39:01.136 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:39:01.209 INFO:teuthology.orchestra.run.smithi155.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T05:39:01.209 INFO:teuthology.orchestra.run.smithi155.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:02.174 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.86.96~20.04.44) ... 2023-10-09T05:39:02.301 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T05:39:02.615 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T05:39:03.342 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-09T05:39:03.459 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for linux-image-5.15.0-86-generic (5.15.0-86.96~20.04.1) ... 2023-10-09T05:39:03.530 INFO:teuthology.orchestra.run.smithi125.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-09T05:39:03.530 INFO:teuthology.orchestra.run.smithi125.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:26.504 INFO:teuthology.orchestra.run.smithi155.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T05:39:26.505 INFO:teuthology.orchestra.run.smithi155.stdout:Sourcing file `/etc/default/grub' 2023-10-09T05:39:26.519 INFO:teuthology.orchestra.run.smithi155.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T05:39:26.551 INFO:teuthology.orchestra.run.smithi155.stdout:Generating grub configuration file ... 2023-10-09T05:39:26.875 INFO:teuthology.orchestra.run.smithi155.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:39:26.899 INFO:teuthology.orchestra.run.smithi155.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:27.302 INFO:teuthology.orchestra.run.smithi155.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T05:39:27.314 INFO:teuthology.orchestra.run.smithi155.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T05:39:28.962 INFO:teuthology.orchestra.run.smithi125.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-09T05:39:28.962 INFO:teuthology.orchestra.run.smithi125.stdout:Sourcing file `/etc/default/grub' 2023-10-09T05:39:28.969 INFO:teuthology.orchestra.run.smithi125.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T05:39:28.989 INFO:teuthology.orchestra.run.smithi125.stdout:Generating grub configuration file ... 2023-10-09T05:39:29.274 INFO:teuthology.orchestra.run.smithi125.stdout:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:39:29.291 INFO:teuthology.orchestra.run.smithi125.stdout:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:29.702 INFO:teuthology.orchestra.run.smithi125.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T05:39:29.714 INFO:teuthology.orchestra.run.smithi125.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T05:39:29.943 INFO:teuthology.orchestra.run.smithi155.stdout:done 2023-10-09T05:39:30.385 DEBUG:teuthology.orchestra.run.smithi155:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T05:39:30.439 INFO:teuthology.orchestra.run.smithi155.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Status: install ok installed 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Priority: optional 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Section: kernel 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Installed-Size: 20 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Architecture: amd64 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Source: linux-meta-hwe-5.15 2023-10-09T05:39:30.440 INFO:teuthology.orchestra.run.smithi155.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout:Recommends: thermald 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout:Description: Generic Linux kernel image 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout: This package will always depend on the latest generic kernel image 2023-10-09T05:39:30.441 INFO:teuthology.orchestra.run.smithi155.stdout: available. 2023-10-09T05:39:30.441 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T05:39:30.441 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T05:39:30.441 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T05:39:30.442 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-09T05:39:30.442 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi155.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T05:39:30.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-get clean 2023-10-09T05:39:30.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-get update 2023-10-09T05:39:30.687 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:39:30.695 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T05:39:30.715 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T05:39:30.800 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T05:39:31.449 INFO:teuthology.orchestra.run.smithi125.stdout:done 2023-10-09T05:39:31.857 DEBUG:teuthology.orchestra.run.smithi125:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T05:39:31.919 INFO:teuthology.orchestra.run.smithi125.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T05:39:31.919 INFO:teuthology.orchestra.run.smithi125.stdout:Status: install ok installed 2023-10-09T05:39:31.919 INFO:teuthology.orchestra.run.smithi125.stdout:Priority: optional 2023-10-09T05:39:31.919 INFO:teuthology.orchestra.run.smithi125.stdout:Section: kernel 2023-10-09T05:39:31.919 INFO:teuthology.orchestra.run.smithi125.stdout:Installed-Size: 20 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Architecture: amd64 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Source: linux-meta-hwe-5.15 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Recommends: thermald 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout:Description: Generic Linux kernel image 2023-10-09T05:39:31.920 INFO:teuthology.orchestra.run.smithi125.stdout: This package will always depend on the latest generic kernel image 2023-10-09T05:39:31.921 INFO:teuthology.orchestra.run.smithi125.stdout: available. 2023-10-09T05:39:31.921 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-86-generic 2023-10-09T05:39:31.921 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-09T05:39:31.921 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-09T05:39:31.921 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-09T05:39:31.921 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi125.front.sepia.ceph.com, path=None, version=distro) 2023-10-09T05:39:31.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-get clean 2023-10-09T05:39:32.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-get update 2023-10-09T05:39:32.276 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:39:32.347 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T05:39:32.406 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:39:32.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T05:39:32.437 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T05:39:32.501 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:39:32.691 INFO:teuthology.orchestra.run.smithi155.stdout:Building dependency tree... 2023-10-09T05:39:32.692 INFO:teuthology.orchestra.run.smithi155.stdout:Reading state information... 2023-10-09T05:39:32.896 INFO:teuthology.orchestra.run.smithi155.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T05:39:32.896 INFO:teuthology.orchestra.run.smithi155.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T05:39:32.897 DEBUG:teuthology.orchestra.run.smithi155:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Status: install ok installed 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Priority: optional 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Section: kernel 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Installed-Size: 20 2023-10-09T05:39:32.915 INFO:teuthology.orchestra.run.smithi155.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Architecture: amd64 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Source: linux-meta-hwe-5.15 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Recommends: thermald 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout:Description: Generic Linux kernel image 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout: This package will always depend on the latest generic kernel image 2023-10-09T05:39:32.916 INFO:teuthology.orchestra.run.smithi155.stdout: available. 2023-10-09T05:39:32.917 DEBUG:teuthology.orchestra.run.smithi155:> mktemp 2023-10-09T05:39:32.922 INFO:teuthology.orchestra.run.smithi155.stdout:/tmp/tmp.AQydU1RfYy 2023-10-09T05:39:32.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo cp /boot/grub/grub.cfg /tmp/tmp.AQydU1RfYy 2023-10-09T05:39:32.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 0666 /tmp/tmp.AQydU1RfYy 2023-10-09T05:39:33.019 DEBUG:teuthology.orchestra.remote:smithi155:/tmp/tmp.AQydU1RfYy is 9KB 2023-10-09T05:39:33.033 DEBUG:teuthology.orchestra.run.smithi155:> rm -fr /tmp/tmp.AQydU1RfYy 2023-10-09T05:39:33.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T05:39:33.093 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:39:33.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T05:39:33.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T05:39:33.169 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T05:39:33.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo update-grub 2023-10-09T05:39:34.503 INFO:teuthology.orchestra.run.smithi155.stderr:Sourcing file `/etc/default/grub' 2023-10-09T05:39:34.510 INFO:teuthology.orchestra.run.smithi155.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T05:39:34.519 INFO:teuthology.orchestra.run.smithi155.stderr:Generating grub configuration file ... 2023-10-09T05:39:34.761 INFO:teuthology.orchestra.run.smithi155.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:39:34.773 INFO:teuthology.orchestra.run.smithi155.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:35.141 INFO:teuthology.orchestra.run.smithi155.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T05:39:35.153 INFO:teuthology.orchestra.run.smithi155.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T05:39:36.767 INFO:teuthology.orchestra.run.smithi155.stderr:done 2023-10-09T05:39:36.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo shutdown -r now 2023-10-09T05:39:40.260 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T05:39:41.570 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:39:41.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-09T05:39:41.664 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:39:41.841 INFO:teuthology.orchestra.run.smithi125.stdout:Building dependency tree... 2023-10-09T05:39:41.842 INFO:teuthology.orchestra.run.smithi125.stdout:Reading state information... 2023-10-09T05:39:42.045 INFO:teuthology.orchestra.run.smithi125.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.86.96~20.04.44). 2023-10-09T05:39:42.046 INFO:teuthology.orchestra.run.smithi125.stdout:0 upgraded, 0 newly installed, 0 to remove and 274 not upgraded. 2023-10-09T05:39:42.047 DEBUG:teuthology.orchestra.run.smithi125:> dpkg -s linux-image-generic-hwe-20.04 2023-10-09T05:39:42.067 INFO:teuthology.orchestra.run.smithi125.stdout:Package: linux-image-generic-hwe-20.04 2023-10-09T05:39:42.067 INFO:teuthology.orchestra.run.smithi125.stdout:Status: install ok installed 2023-10-09T05:39:42.067 INFO:teuthology.orchestra.run.smithi125.stdout:Priority: optional 2023-10-09T05:39:42.067 INFO:teuthology.orchestra.run.smithi125.stdout:Section: kernel 2023-10-09T05:39:42.067 INFO:teuthology.orchestra.run.smithi125.stdout:Installed-Size: 20 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Maintainer: Ubuntu Kernel Team 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Architecture: amd64 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Source: linux-meta-hwe-5.15 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Version: 5.15.0.86.96~20.04.44 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-86), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Depends: linux-image-5.15.0-86-generic, linux-modules-extra-5.15.0-86-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Recommends: thermald 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout:Description: Generic Linux kernel image 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout: This package will always depend on the latest generic kernel image 2023-10-09T05:39:42.068 INFO:teuthology.orchestra.run.smithi125.stdout: available. 2023-10-09T05:39:42.069 DEBUG:teuthology.orchestra.run.smithi125:> mktemp 2023-10-09T05:39:42.074 INFO:teuthology.orchestra.run.smithi125.stdout:/tmp/tmp.g0Yv6Bvf36 2023-10-09T05:39:42.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo cp /boot/grub/grub.cfg /tmp/tmp.g0Yv6Bvf36 2023-10-09T05:39:42.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 0666 /tmp/tmp.g0Yv6Bvf36 2023-10-09T05:39:42.165 DEBUG:teuthology.orchestra.remote:smithi125:/tmp/tmp.g0Yv6Bvf36 is 9KB 2023-10-09T05:39:42.179 DEBUG:teuthology.orchestra.run.smithi125:> rm -fr /tmp/tmp.g0Yv6Bvf36 2023-10-09T05:39:42.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-09T05:39:42.240 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:39:42.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-09T05:39:42.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-09T05:39:42.275 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-86-generic 2023-10-09T05:39:42.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo update-grub 2023-10-09T05:39:43.568 INFO:teuthology.orchestra.run.smithi125.stderr:Sourcing file `/etc/default/grub' 2023-10-09T05:39:43.574 INFO:teuthology.orchestra.run.smithi125.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-09T05:39:43.587 INFO:teuthology.orchestra.run.smithi125.stderr:Generating grub configuration file ... 2023-10-09T05:39:43.860 INFO:teuthology.orchestra.run.smithi125.stderr:Found linux image: /boot/vmlinuz-5.15.0-86-generic 2023-10-09T05:39:43.872 INFO:teuthology.orchestra.run.smithi125.stderr:Found initrd image: /boot/initrd.img-5.15.0-86-generic 2023-10-09T05:39:44.221 INFO:teuthology.orchestra.run.smithi125.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-09T05:39:44.232 INFO:teuthology.orchestra.run.smithi125.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-09T05:39:44.986 INFO:teuthology.orchestra.run.smithi125.stderr:done 2023-10-09T05:39:44.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo shutdown -r now 2023-10-09T05:40:06.790 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-09T05:40:06.790 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:40:06.791 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:40:14.991 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-09T05:40:14.992 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:40:14.992 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:40:25.338 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2023-10-09T05:40:34.346 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:40:34.347 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:40:37.402 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2023-10-09T05:40:49.414 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:40:49.415 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:40:49.658 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.125 2023-10-09T05:40:58.666 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:40:58.667 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:41:07.834 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2023-10-09T05:41:17.050 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.125 2023-10-09T05:41:22.837 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:41:22.837 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:41:23.504 DEBUG:teuthology.orchestra.run.smithi155:> true 2023-10-09T05:41:24.617 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:41:24.618 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-86-generic"... 2023-10-09T05:41:24.618 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2023-10-09T05:41:24.665 INFO:teuthology.orchestra.run.smithi155.stdout:5.15.0-86-generic 2023-10-09T05:41:24.665 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T05:41:24.665 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T05:41:24.665 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T05:41:25.667 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T05:41:25.667 DEBUG:teuthology.orchestra.run.smithi155:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:41:25.754 INFO:teuthology.orchestra.run.smithi155.stdout:ttyS1 2023-10-09T05:41:25.782 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:29.051 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:41:29.052 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:41:29.429 DEBUG:teuthology.orchestra.run.smithi125:> true 2023-10-09T05:41:30.366 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:41:30.366 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-86-generic"... 2023-10-09T05:41:30.367 DEBUG:teuthology.orchestra.run.smithi125:> uname -r 2023-10-09T05:41:30.417 INFO:teuthology.orchestra.run.smithi125.stdout:5.15.0-86-generic 2023-10-09T05:41:30.417 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-86-generic vs 5.15.0-86-generic 2023-10-09T05:41:30.417 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-09T05:41:30.417 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-09T05:41:31.418 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T05:41:31.419 DEBUG:teuthology.orchestra.run.smithi125:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:41:31.722 INFO:teuthology.orchestra.run.smithi125.stdout:ttyS1 2023-10-09T05:41:31.749 DEBUG:teuthology.parallel:result is None 2023-10-09T05:41:31.749 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T05:41:31.756 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T05:41:31.756 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:41:31.759 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:41:31.766 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T05:41:31.806 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T05:41:31.878 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T05:41:31.878 DEBUG:teuthology.orchestra.run.smithi125:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:41:31.880 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:41:31.916 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T05:41:31.922 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T05:41:31.923 DEBUG:teuthology.orchestra.run.smithi125:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:41:31.960 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:41:31.984 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:41:31.987 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:41:31.991 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:41:31.994 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:41:31.996 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T05:41:32.003 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T05:41:32.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:41:32.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:41:32.056 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T05:41:32.064 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T05:41:32.064 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:41:32.092 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:41:32.106 DEBUG:teuthology.orchestra.run.smithi125:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:41:32.145 DEBUG:teuthology.orchestra.run.smithi125:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:41:32.194 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:41:32.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:41:32.265 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:41:32.273 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:41:32.322 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:41:32.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:41:32.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo service rsyslog restart 2023-10-09T05:41:32.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo service rsyslog restart 2023-10-09T05:41:32.484 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T05:41:32.491 INFO:teuthology.task.internal:Starting timer... 2023-10-09T05:41:32.491 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T05:41:32.549 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T05:41:32.683 DEBUG:teuthology.task.selinux:Excluding smithi125: OS 'ubuntu' does not support SELinux 2023-10-09T05:41:32.683 DEBUG:teuthology.task.selinux:Excluding smithi155: OS 'ubuntu' does not support SELinux 2023-10-09T05:41:32.683 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T05:41:32.683 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-09T05:41:32.683 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T05:41:32.683 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T05:41:32.785 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-10-09T05:41:32.785 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T05:41:32.801 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T05:41:32.802 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi125.front.sepia.ceph.com,smithi155.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T05:47:19.735 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi125.front.sepia.ceph.com'), Remote(name='ubuntu@smithi155.front.sepia.ceph.com')] 2023-10-09T05:47:19.736 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:47:19.737 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:47:19.817 DEBUG:teuthology.orchestra.run.smithi125:> true 2023-10-09T05:47:19.932 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi125.front.sepia.ceph.com' 2023-10-09T05:47:19.932 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:47:19.933 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:47:20.021 DEBUG:teuthology.orchestra.run.smithi155:> true 2023-10-09T05:47:20.144 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi155.front.sepia.ceph.com' 2023-10-09T05:47:20.144 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T05:47:20.158 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T05:47:20.158 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:47:20.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:47:20.161 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:47:20.161 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:47:20.192 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T05:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Command line: ntpd -gq 2023-10-09T05:47:20.193 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: proto: precision = 0.100 usec (-23) 2023-10-09T05:47:20.194 INFO:teuthology.orchestra.run.smithi125.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T05:47:20.194 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T05:47:20.194 INFO:teuthology.orchestra.run.smithi125.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T05:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T05:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T05:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T05:47:20.195 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T05:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen normally on 3 enp3s0f1 172.21.15.125:123 2023-10-09T05:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen normally on 4 lo [::1]:123 2023-10-09T05:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d739%5]:123 2023-10-09T05:47:20.196 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:20 ntpd[22680]: Listening on routing socket on fd #22 for interface updates 2023-10-09T05:47:20.214 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-09T05:47:20.214 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Command line: ntpd -gq 2023-10-09T05:47:20.215 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: proto: precision = 0.065 usec (-24) 2023-10-09T05:47:20.215 INFO:teuthology.orchestra.run.smithi155.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T05:47:20.216 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-09T05:47:20.216 INFO:teuthology.orchestra.run.smithi155.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: restrict ::: KOD does nothing without LIMITED. 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen and drop on 0 v6wildcard [::]:123 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen normally on 2 lo 127.0.0.1:123 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen normally on 3 enp3s0f1 172.21.15.155:123 2023-10-09T05:47:20.217 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen normally on 4 lo [::1]:123 2023-10-09T05:47:20.218 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6ded%5]:123 2023-10-09T05:47:20.218 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:20 ntpd[22711]: Listening on routing socket on fd #22 for interface updates 2023-10-09T05:47:21.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:21 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:21.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:21 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:21.215 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:21.215 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:21.215 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:21.215 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:21.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:21.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:21.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:21.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:21 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:22.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:22.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:22.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:22.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:22.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:22.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:22 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:23.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:23.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:23 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:23.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:23.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:23.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:23 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:24.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:24.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:24.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:24.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:24.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:24.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:24 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:25.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:25.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:25 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:25.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:25 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:26.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:26.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:26.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:26.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:26.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:26.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:26 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:27.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:27.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:27 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:27.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.216 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:27.217 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:27.218 INFO:teuthology.orchestra.run.smithi155.stdout: 9 Oct 05:47:27 ntpd[22711]: ntpd: time slew -0.000182 s 2023-10-09T05:47:27.218 INFO:teuthology.orchestra.run.smithi155.stdout:ntpd: time slew -0.000182s 2023-10-09T05:47:27.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T05:47:27.218 INFO:teuthology.orchestra.run.smithi155.stderr: 9 Oct 05:47:27 ntpd[22711]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout:============================================================================== 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:27.278 INFO:teuthology.orchestra.run.smithi155.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:28.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:28.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:28.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-09T05:47:28.193 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: can't open /var/log/ntpstats/rawstats.20231009: Permission denied 2023-10-09T05:47:28.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-09T05:47:28.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: can't open /var/log/ntpstats/peerstats.20231009: Permission denied 2023-10-09T05:47:28.194 INFO:teuthology.orchestra.run.smithi125.stdout: 9 Oct 05:47:28 ntpd[22680]: ntpd: time slew +0.008292 s 2023-10-09T05:47:28.194 INFO:teuthology.orchestra.run.smithi125.stdout:ntpd: time slew +0.008292s 2023-10-09T05:47:28.194 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-09T05:47:28.195 INFO:teuthology.orchestra.run.smithi125.stderr: 9 Oct 05:47:28 ntpd[22680]: can't open /var/log/ntpstats/loopstats.20231009: Permission denied 2023-10-09T05:47:28.250 INFO:teuthology.orchestra.run.smithi125.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T05:47:28.250 INFO:teuthology.orchestra.run.smithi125.stdout:============================================================================== 2023-10-09T05:47:28.250 INFO:teuthology.orchestra.run.smithi125.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:28.250 INFO:teuthology.orchestra.run.smithi125.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:28.250 INFO:teuthology.orchestra.run.smithi125.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:28.251 INFO:teuthology.orchestra.run.smithi125.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-09T05:47:28.252 INFO:teuthology.run_tasks:Running task install... 2023-10-09T05:47:28.261 DEBUG:teuthology.task.install:project ceph 2023-10-09T05:47:28.261 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T05:47:28.261 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:47:28.261 INFO:teuthology.task.install:Using flavor: default 2023-10-09T05:47:28.267 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T05:47:28.267 INFO:teuthology.task.install:extra packages: [] 2023-10-09T05:47:28.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-key list | grep Ceph 2023-10-09T05:47:28.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-key list | grep Ceph 2023-10-09T05:47:28.371 INFO:teuthology.orchestra.run.smithi155.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T05:47:28.392 INFO:teuthology.orchestra.run.smithi125.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-09T05:47:28.434 INFO:teuthology.orchestra.run.smithi155.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T05:47:28.435 INFO:teuthology.orchestra.run.smithi155.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T05:47:28.436 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T05:47:28.436 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:47:28.436 INFO:teuthology.packaging:ref: None 2023-10-09T05:47:28.436 INFO:teuthology.packaging:tag: None 2023-10-09T05:47:28.436 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:47:28.436 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:47:28.436 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T05:47:28.450 INFO:teuthology.orchestra.run.smithi125.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-09T05:47:28.450 INFO:teuthology.orchestra.run.smithi125.stdout:uid [ unknown] Ceph.com (release key) 2023-10-09T05:47:28.452 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-09T05:47:28.452 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:47:28.452 INFO:teuthology.packaging:ref: None 2023-10-09T05:47:28.452 INFO:teuthology.packaging:tag: None 2023-10-09T05:47:28.453 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:47:28.453 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:47:28.453 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T05:47:28.769 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T05:47:28.769 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T05:47:28.778 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default/ 2023-10-09T05:47:28.778 INFO:teuthology.task.install.deb:Package version is 16.2.14-48-gaeb2e425-1focal 2023-10-09T05:47:28.943 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:47:28.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T05:47:28.949 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:47:28.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-09T05:47:28.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo apt-get update 2023-10-09T05:47:28.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo apt-get update 2023-10-09T05:47:29.151 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:47:29.151 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T05:47:29.159 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T05:47:29.179 INFO:teuthology.orchestra.run.smithi125.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T05:47:29.206 INFO:teuthology.orchestra.run.smithi155.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T05:47:29.228 INFO:teuthology.orchestra.run.smithi155.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T05:47:29.252 INFO:teuthology.orchestra.run.smithi155.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T05:47:29.259 INFO:teuthology.orchestra.run.smithi125.stdout:Ign:5 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal InRelease 2023-10-09T05:47:29.279 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-09T05:47:29.279 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:5 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-09T05:47:29.279 INFO:teuthology.orchestra.run.smithi155.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T05:47:29.295 INFO:teuthology.orchestra.run.smithi125.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-09T05:47:29.320 INFO:teuthology.orchestra.run.smithi155.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T05:47:29.333 INFO:teuthology.orchestra.run.smithi125.stdout:Ign:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal Release.gpg 2023-10-09T05:47:29.349 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-09T05:47:29.369 INFO:teuthology.orchestra.run.smithi125.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-10-09T05:47:29.431 INFO:teuthology.orchestra.run.smithi155.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-09T05:47:29.431 INFO:teuthology.orchestra.run.smithi125.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-09T05:47:29.781 INFO:teuthology.orchestra.run.smithi125.stdout:Fetched 28.0 kB in 1s (45.6 kB/s) 2023-10-09T05:47:29.887 INFO:teuthology.orchestra.run.smithi155.stdout:Fetched 28.0 kB in 1s (38.9 kB/s) 2023-10-09T05:47:30.825 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:47:30.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T05:47:30.931 INFO:teuthology.orchestra.run.smithi125.stdout:Reading package lists... 2023-10-09T05:47:30.949 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:47:30.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-48-gaeb2e425-1focal cephadm=16.2.14-48-gaeb2e425-1focal ceph-mds=16.2.14-48-gaeb2e425-1focal ceph-mgr=16.2.14-48-gaeb2e425-1focal ceph-common=16.2.14-48-gaeb2e425-1focal ceph-fuse=16.2.14-48-gaeb2e425-1focal ceph-test=16.2.14-48-gaeb2e425-1focal radosgw=16.2.14-48-gaeb2e425-1focal python3-rados=16.2.14-48-gaeb2e425-1focal python3-rgw=16.2.14-48-gaeb2e425-1focal python3-cephfs=16.2.14-48-gaeb2e425-1focal python3-rbd=16.2.14-48-gaeb2e425-1focal libcephfs2=16.2.14-48-gaeb2e425-1focal libcephfs-dev=16.2.14-48-gaeb2e425-1focal librados2=16.2.14-48-gaeb2e425-1focal librbd1=16.2.14-48-gaeb2e425-1focal rbd-fuse=16.2.14-48-gaeb2e425-1focal 2023-10-09T05:47:31.042 INFO:teuthology.orchestra.run.smithi155.stdout:Reading package lists... 2023-10-09T05:47:31.132 INFO:teuthology.orchestra.run.smithi125.stdout:Building dependency tree... 2023-10-09T05:47:31.133 INFO:teuthology.orchestra.run.smithi125.stdout:Reading state information... 2023-10-09T05:47:31.243 INFO:teuthology.orchestra.run.smithi155.stdout:Building dependency tree... 2023-10-09T05:47:31.244 INFO:teuthology.orchestra.run.smithi155.stdout:Reading state information... 2023-10-09T05:47:31.310 INFO:teuthology.orchestra.run.smithi125.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T05:47:31.311 INFO:teuthology.orchestra.run.smithi125.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T05:47:31.311 INFO:teuthology.orchestra.run.smithi125.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T05:47:31.311 INFO:teuthology.orchestra.run.smithi125.stdout:The following additional packages will be installed: 2023-10-09T05:47:31.311 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T05:47:31.312 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T05:47:31.312 INFO:teuthology.orchestra.run.smithi125.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T05:47:31.312 INFO:teuthology.orchestra.run.smithi125.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T05:47:31.312 INFO:teuthology.orchestra.run.smithi125.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T05:47:31.312 INFO:teuthology.orchestra.run.smithi125.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T05:47:31.313 INFO:teuthology.orchestra.run.smithi125.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout:Suggested packages: 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T05:47:31.314 INFO:teuthology.orchestra.run.smithi125.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T05:47:31.315 INFO:teuthology.orchestra.run.smithi125.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T05:47:31.315 INFO:teuthology.orchestra.run.smithi125.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T05:47:31.315 INFO:teuthology.orchestra.run.smithi125.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T05:47:31.315 INFO:teuthology.orchestra.run.smithi125.stdout:Recommended packages: 2023-10-09T05:47:31.315 INFO:teuthology.orchestra.run.smithi125.stdout: btrfs-tools 2023-10-09T05:47:31.413 INFO:teuthology.orchestra.run.smithi155.stdout:The following packages were automatically installed and are no longer required: 2023-10-09T05:47:31.414 INFO:teuthology.orchestra.run.smithi155.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-09T05:47:31.414 INFO:teuthology.orchestra.run.smithi155.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-09T05:47:31.414 INFO:teuthology.orchestra.run.smithi155.stdout:The following additional packages will be installed: 2023-10-09T05:47:31.414 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-09T05:47:31.415 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-09T05:47:31.415 INFO:teuthology.orchestra.run.smithi155.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi155.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi155.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi155.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-09T05:47:31.416 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-09T05:47:31.417 INFO:teuthology.orchestra.run.smithi155.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-09T05:47:31.418 INFO:teuthology.orchestra.run.smithi155.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-09T05:47:31.418 INFO:teuthology.orchestra.run.smithi155.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-09T05:47:31.418 INFO:teuthology.orchestra.run.smithi155.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-09T05:47:31.418 INFO:teuthology.orchestra.run.smithi155.stdout:Suggested packages: 2023-10-09T05:47:31.418 INFO:teuthology.orchestra.run.smithi155.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-09T05:47:31.419 INFO:teuthology.orchestra.run.smithi155.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-09T05:47:31.420 INFO:teuthology.orchestra.run.smithi155.stdout:Recommended packages: 2023-10-09T05:47:31.420 INFO:teuthology.orchestra.run.smithi155.stdout: btrfs-tools 2023-10-09T05:47:31.426 INFO:teuthology.orchestra.run.smithi125.stdout:The following NEW packages will be installed: 2023-10-09T05:47:31.426 INFO:teuthology.orchestra.run.smithi125.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T05:47:31.426 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T05:47:31.427 INFO:teuthology.orchestra.run.smithi125.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T05:47:31.427 INFO:teuthology.orchestra.run.smithi125.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T05:47:31.428 INFO:teuthology.orchestra.run.smithi125.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T05:47:31.429 INFO:teuthology.orchestra.run.smithi125.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T05:47:31.430 INFO:teuthology.orchestra.run.smithi125.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T05:47:31.430 INFO:teuthology.orchestra.run.smithi125.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T05:47:31.430 INFO:teuthology.orchestra.run.smithi125.stdout:The following packages will be upgraded: 2023-10-09T05:47:31.430 INFO:teuthology.orchestra.run.smithi125.stdout: librados2 librbd1 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout:The following NEW packages will be installed: 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-09T05:47:31.525 INFO:teuthology.orchestra.run.smithi155.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-09T05:47:31.527 INFO:teuthology.orchestra.run.smithi155.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-09T05:47:31.527 INFO:teuthology.orchestra.run.smithi155.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-09T05:47:31.527 INFO:teuthology.orchestra.run.smithi155.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-09T05:47:31.527 INFO:teuthology.orchestra.run.smithi155.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-09T05:47:31.527 INFO:teuthology.orchestra.run.smithi155.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-09T05:47:31.528 INFO:teuthology.orchestra.run.smithi155.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-09T05:47:31.528 INFO:teuthology.orchestra.run.smithi155.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-09T05:47:31.528 INFO:teuthology.orchestra.run.smithi155.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-09T05:47:31.528 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-09T05:47:31.528 INFO:teuthology.orchestra.run.smithi155.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-09T05:47:31.529 INFO:teuthology.orchestra.run.smithi155.stdout:The following packages will be upgraded: 2023-10-09T05:47:31.530 INFO:teuthology.orchestra.run.smithi155.stdout: librados2 librbd1 2023-10-09T05:47:31.539 INFO:teuthology.orchestra.run.smithi125.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T05:47:31.539 INFO:teuthology.orchestra.run.smithi125.stdout:Need to get 174 MB of archives. 2023-10-09T05:47:31.539 INFO:teuthology.orchestra.run.smithi125.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T05:47:31.539 INFO:teuthology.orchestra.run.smithi125.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T05:47:31.655 INFO:teuthology.orchestra.run.smithi155.stdout:2 upgraded, 98 newly installed, 0 to remove and 256 not upgraded. 2023-10-09T05:47:31.655 INFO:teuthology.orchestra.run.smithi155.stdout:Need to get 174 MB of archives. 2023-10-09T05:47:31.655 INFO:teuthology.orchestra.run.smithi155.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-09T05:47:31.656 INFO:teuthology.orchestra.run.smithi155.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T05:47:31.692 INFO:teuthology.orchestra.run.smithi125.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-48-gaeb2e425-1focal [3,960 B] 2023-10-09T05:47:31.698 INFO:teuthology.orchestra.run.smithi155.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T05:47:31.718 INFO:teuthology.orchestra.run.smithi155.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-09T05:47:31.726 INFO:teuthology.orchestra.run.smithi125.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T05:47:31.730 INFO:teuthology.orchestra.run.smithi125.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T05:47:31.754 INFO:teuthology.orchestra.run.smithi125.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T05:47:31.766 INFO:teuthology.orchestra.run.smithi125.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T05:47:31.786 INFO:teuthology.orchestra.run.smithi125.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-48-gaeb2e425-1focal [3,383 kB] 2023-10-09T05:47:31.800 INFO:teuthology.orchestra.run.smithi125.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T05:47:31.809 INFO:teuthology.orchestra.run.smithi125.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T05:47:31.812 INFO:teuthology.orchestra.run.smithi125.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T05:47:31.822 INFO:teuthology.orchestra.run.smithi125.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T05:47:31.824 INFO:teuthology.orchestra.run.smithi125.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T05:47:31.825 INFO:teuthology.orchestra.run.smithi125.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T05:47:31.830 INFO:teuthology.orchestra.run.smithi125.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T05:47:31.830 INFO:teuthology.orchestra.run.smithi125.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T05:47:31.831 INFO:teuthology.orchestra.run.smithi125.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T05:47:31.831 INFO:teuthology.orchestra.run.smithi125.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T05:47:31.850 INFO:teuthology.orchestra.run.smithi125.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T05:47:31.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T05:47:31.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T05:47:31.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T05:47:31.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T05:47:31.869 INFO:teuthology.orchestra.run.smithi125.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T05:47:31.872 INFO:teuthology.orchestra.run.smithi125.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T05:47:31.873 INFO:teuthology.orchestra.run.smithi125.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T05:47:31.881 INFO:teuthology.orchestra.run.smithi125.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T05:47:31.881 INFO:teuthology.orchestra.run.smithi125.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T05:47:31.890 INFO:teuthology.orchestra.run.smithi125.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T05:47:31.890 INFO:teuthology.orchestra.run.smithi125.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T05:47:31.892 INFO:teuthology.orchestra.run.smithi125.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T05:47:31.895 INFO:teuthology.orchestra.run.smithi125.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T05:47:31.897 INFO:teuthology.orchestra.run.smithi125.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T05:47:31.906 INFO:teuthology.orchestra.run.smithi155.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T05:47:31.909 INFO:teuthology.orchestra.run.smithi125.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T05:47:31.911 INFO:teuthology.orchestra.run.smithi125.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T05:47:31.913 INFO:teuthology.orchestra.run.smithi125.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T05:47:31.915 INFO:teuthology.orchestra.run.smithi125.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T05:47:31.916 INFO:teuthology.orchestra.run.smithi125.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T05:47:31.928 INFO:teuthology.orchestra.run.smithi125.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T05:47:32.054 INFO:teuthology.orchestra.run.smithi155.stdout:Get:5 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T05:47:32.084 INFO:teuthology.orchestra.run.smithi155.stdout:Get:6 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T05:47:32.094 INFO:teuthology.orchestra.run.smithi125.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T05:47:32.102 INFO:teuthology.orchestra.run.smithi155.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T05:47:32.102 INFO:teuthology.orchestra.run.smithi155.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T05:47:32.110 INFO:teuthology.orchestra.run.smithi125.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T05:47:32.111 INFO:teuthology.orchestra.run.smithi125.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T05:47:32.112 INFO:teuthology.orchestra.run.smithi155.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T05:47:32.116 INFO:teuthology.orchestra.run.smithi155.stdout:Get:10 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T05:47:32.116 INFO:teuthology.orchestra.run.smithi155.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-09T05:47:32.123 INFO:teuthology.orchestra.run.smithi155.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-09T05:47:32.128 INFO:teuthology.orchestra.run.smithi125.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi125.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi125.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi125.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi155.stdout:Get:13 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi125.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T05:47:32.132 INFO:teuthology.orchestra.run.smithi125.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T05:47:32.132 INFO:teuthology.orchestra.run.smithi125.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T05:47:32.140 INFO:teuthology.orchestra.run.smithi125.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T05:47:32.140 INFO:teuthology.orchestra.run.smithi125.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T05:47:32.159 INFO:teuthology.orchestra.run.smithi125.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T05:47:32.164 INFO:teuthology.orchestra.run.smithi125.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T05:47:32.167 INFO:teuthology.orchestra.run.smithi125.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T05:47:32.168 INFO:teuthology.orchestra.run.smithi155.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-09T05:47:32.169 INFO:teuthology.orchestra.run.smithi125.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T05:47:32.170 INFO:teuthology.orchestra.run.smithi125.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T05:47:32.173 INFO:teuthology.orchestra.run.smithi125.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T05:47:32.175 INFO:teuthology.orchestra.run.smithi125.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T05:47:32.178 INFO:teuthology.orchestra.run.smithi125.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T05:47:32.179 INFO:teuthology.orchestra.run.smithi125.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T05:47:32.179 INFO:teuthology.orchestra.run.smithi125.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T05:47:32.181 INFO:teuthology.orchestra.run.smithi155.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-09T05:47:32.198 INFO:teuthology.orchestra.run.smithi125.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T05:47:32.200 INFO:teuthology.orchestra.run.smithi125.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T05:47:32.217 INFO:teuthology.orchestra.run.smithi125.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T05:47:32.217 INFO:teuthology.orchestra.run.smithi125.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T05:47:32.218 INFO:teuthology.orchestra.run.smithi125.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T05:47:32.218 INFO:teuthology.orchestra.run.smithi125.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T05:47:32.220 INFO:teuthology.orchestra.run.smithi125.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T05:47:32.221 INFO:teuthology.orchestra.run.smithi125.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T05:47:32.221 INFO:teuthology.orchestra.run.smithi125.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T05:47:32.221 INFO:teuthology.orchestra.run.smithi125.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T05:47:32.239 INFO:teuthology.orchestra.run.smithi125.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T05:47:32.268 INFO:teuthology.orchestra.run.smithi125.stdout:Get:72 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-48-gaeb2e425-1focal [3,391 kB] 2023-10-09T05:47:32.307 INFO:teuthology.orchestra.run.smithi155.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-09T05:47:32.308 INFO:teuthology.orchestra.run.smithi155.stdout:Get:17 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T05:47:32.312 INFO:teuthology.orchestra.run.smithi155.stdout:Get:18 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T05:47:32.330 INFO:teuthology.orchestra.run.smithi155.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-09T05:47:32.336 INFO:teuthology.orchestra.run.smithi155.stdout:Get:20 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T05:47:32.337 INFO:teuthology.orchestra.run.smithi155.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-09T05:47:32.367 INFO:teuthology.orchestra.run.smithi155.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-09T05:47:32.372 INFO:teuthology.orchestra.run.smithi155.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-09T05:47:32.375 INFO:teuthology.orchestra.run.smithi155.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-09T05:47:32.554 INFO:teuthology.orchestra.run.smithi155.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-09T05:47:32.621 INFO:teuthology.orchestra.run.smithi125.stdout:Get:73 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-48-gaeb2e425-1focal [730 kB] 2023-10-09T05:47:32.701 INFO:teuthology.orchestra.run.smithi125.stdout:Get:74 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-48-gaeb2e425-1focal [413 kB] 2023-10-09T05:47:32.721 INFO:teuthology.orchestra.run.smithi155.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-09T05:47:32.744 INFO:teuthology.orchestra.run.smithi125.stdout:Get:75 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-48-gaeb2e425-1focal [22.2 kB] 2023-10-09T05:47:32.747 INFO:teuthology.orchestra.run.smithi125.stdout:Get:76 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-48-gaeb2e425-1focal [212 kB] 2023-10-09T05:47:32.752 INFO:teuthology.orchestra.run.smithi155.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-09T05:47:32.769 INFO:teuthology.orchestra.run.smithi125.stdout:Get:77 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-48-gaeb2e425-1focal [38.4 kB] 2023-10-09T05:47:32.774 INFO:teuthology.orchestra.run.smithi125.stdout:Get:78 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-48-gaeb2e425-1focal [386 kB] 2023-10-09T05:47:32.775 INFO:teuthology.orchestra.run.smithi155.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-09T05:47:32.814 INFO:teuthology.orchestra.run.smithi125.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-48-gaeb2e425-1focal [3,689 kB] 2023-10-09T05:47:32.891 INFO:teuthology.orchestra.run.smithi155.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-09T05:47:32.904 INFO:teuthology.orchestra.run.smithi155.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-09T05:47:32.908 INFO:teuthology.orchestra.run.smithi155.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-09T05:47:32.975 INFO:teuthology.orchestra.run.smithi155.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-09T05:47:32.991 INFO:teuthology.orchestra.run.smithi155.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-09T05:47:33.026 INFO:teuthology.orchestra.run.smithi155.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-09T05:47:33.052 INFO:teuthology.orchestra.run.smithi155.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-09T05:47:33.162 INFO:teuthology.orchestra.run.smithi125.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-48-gaeb2e425-1focal [116 kB] 2023-10-09T05:47:33.172 INFO:teuthology.orchestra.run.smithi125.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-48-gaeb2e425-1focal [448 kB] 2023-10-09T05:47:33.208 INFO:teuthology.orchestra.run.smithi125.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-48-gaeb2e425-1focal [22.6 MB] 2023-10-09T05:47:33.220 INFO:teuthology.orchestra.run.smithi155.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-09T05:47:33.680 INFO:teuthology.orchestra.run.smithi155.stdout:Get:37 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T05:47:33.762 INFO:teuthology.orchestra.run.smithi155.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-09T05:47:33.762 INFO:teuthology.orchestra.run.smithi155.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-09T05:47:33.765 INFO:teuthology.orchestra.run.smithi155.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-09T05:47:33.769 INFO:teuthology.orchestra.run.smithi155.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-09T05:47:33.780 INFO:teuthology.orchestra.run.smithi155.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-09T05:47:33.803 INFO:teuthology.orchestra.run.smithi155.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-09T05:47:33.816 INFO:teuthology.orchestra.run.smithi155.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-09T05:47:33.817 INFO:teuthology.orchestra.run.smithi155.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-09T05:47:33.852 INFO:teuthology.orchestra.run.smithi155.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-09T05:47:33.864 INFO:teuthology.orchestra.run.smithi155.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-09T05:47:34.035 INFO:teuthology.orchestra.run.smithi155.stdout:Get:48 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T05:47:34.041 INFO:teuthology.orchestra.run.smithi155.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-09T05:47:34.048 INFO:teuthology.orchestra.run.smithi155.stdout:Get:50 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T05:47:34.058 INFO:teuthology.orchestra.run.smithi155.stdout:Get:51 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T05:47:34.124 INFO:teuthology.orchestra.run.smithi155.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-09T05:47:34.195 INFO:teuthology.orchestra.run.smithi155.stdout:Get:53 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T05:47:34.208 INFO:teuthology.orchestra.run.smithi155.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-09T05:47:34.839 INFO:teuthology.orchestra.run.smithi155.stdout:Get:55 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T05:47:34.876 INFO:teuthology.orchestra.run.smithi125.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-48-gaeb2e425-1focal [5,916 kB] 2023-10-09T05:47:34.945 INFO:teuthology.orchestra.run.smithi155.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-09T05:47:35.020 INFO:teuthology.orchestra.run.smithi155.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-09T05:47:35.022 INFO:teuthology.orchestra.run.smithi155.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-09T05:47:35.062 INFO:teuthology.orchestra.run.smithi155.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-09T05:47:35.062 INFO:teuthology.orchestra.run.smithi155.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-09T05:47:35.063 INFO:teuthology.orchestra.run.smithi155.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-09T05:47:35.063 INFO:teuthology.orchestra.run.smithi155.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-09T05:47:35.064 INFO:teuthology.orchestra.run.smithi155.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-09T05:47:35.176 INFO:teuthology.orchestra.run.smithi125.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-48-gaeb2e425-1focal [204 kB] 2023-10-09T05:47:35.188 INFO:teuthology.orchestra.run.smithi125.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-48-gaeb2e425-1focal [126 kB] 2023-10-09T05:47:35.194 INFO:teuthology.orchestra.run.smithi125.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-48-gaeb2e425-1focal [1,536 kB] 2023-10-09T05:47:35.227 INFO:teuthology.orchestra.run.smithi155.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-09T05:47:35.290 INFO:teuthology.orchestra.run.smithi125.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-48-gaeb2e425-1focal [6,993 kB] 2023-10-09T05:47:35.310 INFO:teuthology.orchestra.run.smithi155.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-09T05:47:35.490 INFO:teuthology.orchestra.run.smithi155.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-09T05:47:35.506 INFO:teuthology.orchestra.run.smithi155.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-09T05:47:35.538 INFO:teuthology.orchestra.run.smithi155.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-09T05:47:35.668 INFO:teuthology.orchestra.run.smithi155.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-09T05:47:35.693 INFO:teuthology.orchestra.run.smithi125.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-48-gaeb2e425-1focal [26.2 MB] 2023-10-09T05:47:35.704 INFO:teuthology.orchestra.run.smithi155.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-09T05:47:35.728 INFO:teuthology.orchestra.run.smithi155.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-09T05:47:35.736 INFO:teuthology.orchestra.run.smithi155.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-09T05:47:35.779 INFO:teuthology.orchestra.run.smithi155.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-09T05:47:35.805 INFO:teuthology.orchestra.run.smithi155.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-09T05:47:35.992 INFO:teuthology.orchestra.run.smithi155.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-09T05:47:36.076 INFO:teuthology.orchestra.run.smithi155.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-09T05:47:36.189 INFO:teuthology.orchestra.run.smithi155.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-09T05:47:36.203 INFO:teuthology.orchestra.run.smithi155.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-09T05:47:36.358 INFO:teuthology.orchestra.run.smithi155.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-09T05:47:36.661 INFO:teuthology.orchestra.run.smithi155.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-09T05:47:36.663 INFO:teuthology.orchestra.run.smithi155.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-09T05:47:36.667 INFO:teuthology.orchestra.run.smithi155.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-09T05:47:36.668 INFO:teuthology.orchestra.run.smithi155.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-09T05:47:36.677 INFO:teuthology.orchestra.run.smithi155.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-09T05:47:36.683 INFO:teuthology.orchestra.run.smithi155.stdout:Get:85 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-09T05:47:36.847 INFO:teuthology.orchestra.run.smithi155.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-09T05:47:36.985 INFO:teuthology.orchestra.run.smithi155.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T05:47:36.985 INFO:teuthology.orchestra.run.smithi155.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T05:47:36.991 INFO:teuthology.orchestra.run.smithi155.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T05:47:37.066 INFO:teuthology.orchestra.run.smithi155.stdout:Get:90 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-09T05:47:37.176 INFO:teuthology.orchestra.run.smithi155.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T05:47:37.183 INFO:teuthology.orchestra.run.smithi155.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T05:47:37.194 INFO:teuthology.orchestra.run.smithi155.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T05:47:37.254 INFO:teuthology.orchestra.run.smithi125.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-48-gaeb2e425-1focal [4,056 B] 2023-10-09T05:47:37.254 INFO:teuthology.orchestra.run.smithi125.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-48-gaeb2e425-1focal [834 kB] 2023-10-09T05:47:37.260 INFO:teuthology.orchestra.run.smithi125.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-48-gaeb2e425-1focal [2,364 kB] 2023-10-09T05:47:37.319 INFO:teuthology.orchestra.run.smithi155.stdout:Get:94 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-09T05:47:37.339 INFO:teuthology.orchestra.run.smithi125.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-48-gaeb2e425-1focal [76.4 kB] 2023-10-09T05:47:37.341 INFO:teuthology.orchestra.run.smithi125.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-48-gaeb2e425-1focal [100 kB] 2023-10-09T05:47:37.347 INFO:teuthology.orchestra.run.smithi125.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-48-gaeb2e425-1focal [2,823 kB] 2023-10-09T05:47:37.406 INFO:teuthology.orchestra.run.smithi155.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T05:47:37.506 INFO:teuthology.orchestra.run.smithi125.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-48-gaeb2e425-1focal [7,865 kB] 2023-10-09T05:47:37.939 INFO:teuthology.orchestra.run.smithi125.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T05:47:37.939 INFO:teuthology.orchestra.run.smithi125.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T05:47:38.043 INFO:teuthology.orchestra.run.smithi155.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-48-gaeb2e425-1focal [4,172 B] 2023-10-09T05:47:38.044 INFO:teuthology.orchestra.run.smithi155.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-48-gaeb2e425-1focal [49.3 MB] 2023-10-09T05:47:40.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T05:47:40.865 INFO:teuthology.orchestra.run.smithi125.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T05:47:41.290 INFO:teuthology.orchestra.run.smithi125.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T05:47:41.813 INFO:teuthology.orchestra.run.smithi155.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-48-gaeb2e425-1focal [21.5 kB] 2023-10-09T05:47:41.813 INFO:teuthology.orchestra.run.smithi155.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-48-gaeb2e425-1focal [11.3 MB] 2023-10-09T05:47:41.940 INFO:teuthology.orchestra.run.smithi125.stdout:Fetched 174 MB in 10s (17.6 MB/s) 2023-10-09T05:47:42.035 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libjaeger. 2023-10-09T05:47:42.081 INFO:teuthology.orchestra.run.smithi125.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T05:47:42.084 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:42.099 INFO:teuthology.orchestra.run.smithi155.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/pacific/aeb2e425bccd1202ba613f25c784b52a47922a5f/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-48-gaeb2e425-1focal [78.4 kB] 2023-10-09T05:47:42.134 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:42.453 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T05:47:42.466 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T05:47:42.545 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T05:47:42.746 INFO:teuthology.orchestra.run.smithi155.stdout:Fetched 174 MB in 11s (16.4 MB/s) 2023-10-09T05:47:42.843 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libjaeger. 2023-10-09T05:47:42.890 INFO:teuthology.orchestra.run.smithi155.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-10-09T05:47:42.893 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../00-libjaeger_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:42.911 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:42.933 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:43.065 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T05:47:43.269 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-09T05:47:43.282 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-09T05:47:43.386 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T05:47:43.765 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:43.785 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../02-librbd1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:43.886 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T05:47:43.931 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking librbd1 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T05:47:44.573 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T05:47:44.587 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:44.640 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:44.675 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../03-librados2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:44.811 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking librados2 (16.2.14-48-gaeb2e425-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-09T05:47:45.026 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-rados. 2023-10-09T05:47:45.039 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:45.084 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:45.438 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T05:47:45.451 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:45.496 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:45.523 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libcephfs2. 2023-10-09T05:47:45.536 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:45.590 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:45.799 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T05:47:45.814 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:45.857 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:45.959 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-rados. 2023-10-09T05:47:45.972 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../05-python3-rados_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:46.017 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:46.194 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T05:47:46.207 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:46.252 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:46.371 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-09T05:47:46.385 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:46.430 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:46.561 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T05:47:46.575 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T05:47:46.619 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T05:47:46.746 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-cephfs. 2023-10-09T05:47:46.762 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:46.804 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:46.846 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T05:47:46.862 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:46.904 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:47.149 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-09T05:47:47.163 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:47.216 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:47.266 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T05:47:47.281 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T05:47:47.324 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T05:47:47.619 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-prettytable. 2023-10-09T05:47:47.632 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-09T05:47:47.686 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T05:47:47.701 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T05:47:47.719 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-09T05:47:47.744 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T05:47:48.064 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T05:47:48.078 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T05:47:48.122 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T05:47:48.138 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-rbd. 2023-10-09T05:47:48.154 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:48.197 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:48.558 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-09T05:47:48.571 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-09T05:47:48.616 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T05:47:48.733 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package librgw2. 2023-10-09T05:47:48.746 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:48.783 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:48.969 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-09T05:47:48.982 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-09T05:47:49.027 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T05:47:49.380 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-09T05:47:49.393 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-09T05:47:49.394 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T05:47:49.410 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:49.430 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T05:47:49.453 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:49.755 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T05:47:49.768 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T05:47:49.775 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package librgw2. 2023-10-09T05:47:49.788 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../14-librgw2_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:49.814 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T05:47:49.841 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:50.225 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package liboath0. 2023-10-09T05:47:50.238 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T05:47:50.283 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T05:47:50.429 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-rgw. 2023-10-09T05:47:50.442 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:50.493 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:50.636 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T05:47:50.649 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:50.694 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:50.863 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-09T05:47:50.876 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-09T05:47:50.921 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T05:47:51.148 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-common. 2023-10-09T05:47:51.160 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:51.206 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:51.283 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package liboath0. 2023-10-09T05:47:51.296 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-09T05:47:51.332 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-09T05:47:51.718 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libradosstriper1. 2023-10-09T05:47:51.734 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:51.785 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:52.305 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-common. 2023-10-09T05:47:52.318 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../19-ceph-common_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:52.355 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:53.411 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-base. 2023-10-09T05:47:53.426 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:53.479 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:54.500 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T05:47:54.513 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T05:47:54.558 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T05:47:54.611 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-base. 2023-10-09T05:47:54.624 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../20-ceph-base_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:47:54.679 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:55.012 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T05:47:55.025 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T05:47:55.070 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T05:47:55.348 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T05:47:55.361 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:55.398 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:55.674 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-cryptography. 2023-10-09T05:47:55.688 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-09T05:47:55.727 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T05:47:55.740 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T05:47:55.742 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T05:47:55.786 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T05:47:56.088 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T05:47:56.103 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T05:47:56.146 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T05:47:56.161 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-openssl. 2023-10-09T05:47:56.174 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-09T05:47:56.227 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-09T05:47:56.508 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T05:47:56.523 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T05:47:56.524 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-09T05:47:56.537 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:47:56.566 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T05:47:56.573 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:47:56.878 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T05:47:56.894 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T05:47:56.937 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T05:47:56.937 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-09T05:47:56.951 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-09T05:47:56.995 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T05:47:57.198 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T05:47:57.211 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T05:47:57.239 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T05:47:57.307 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-09T05:47:57.320 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-09T05:47:57.357 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-09T05:47:57.542 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-mako. 2023-10-09T05:47:57.556 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T05:47:57.600 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T05:47:57.694 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-jwt. 2023-10-09T05:47:57.708 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-09T05:47:57.752 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T05:47:57.904 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T05:47:57.917 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T05:47:57.953 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T05:47:58.031 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-logutils. 2023-10-09T05:47:58.044 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-09T05:47:58.081 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-09T05:47:58.232 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T05:47:58.246 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T05:47:58.281 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T05:47:58.409 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-09T05:47:58.424 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-09T05:47:58.467 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-09T05:47:58.550 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-webob. 2023-10-09T05:47:58.566 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T05:47:58.608 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T05:47:58.770 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-mako. 2023-10-09T05:47:58.786 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-09T05:47:58.828 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T05:47:58.945 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T05:47:58.960 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T05:47:59.003 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T05:47:59.115 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-09T05:47:59.129 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-09T05:47:59.173 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-09T05:47:59.322 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T05:47:59.338 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T05:47:59.380 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T05:47:59.460 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-09T05:47:59.473 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-09T05:47:59.518 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T05:47:59.709 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T05:47:59.723 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T05:47:59.770 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T05:47:59.837 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-webob. 2023-10-09T05:47:59.851 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-09T05:47:59.896 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-09T05:48:00.120 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T05:48:00.134 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T05:48:00.178 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T05:48:00.241 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-09T05:48:00.255 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-09T05:48:00.299 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T05:48:00.473 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-paste. 2023-10-09T05:48:00.487 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T05:48:00.531 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T05:48:00.585 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-bs4. 2023-10-09T05:48:00.599 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-09T05:48:00.643 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-09T05:48:00.842 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T05:48:00.858 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T05:48:00.903 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T05:48:00.997 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-waitress. 2023-10-09T05:48:01.010 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-09T05:48:01.058 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T05:48:01.205 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T05:48:01.219 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T05:48:01.264 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T05:48:01.400 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-tempita. 2023-10-09T05:48:01.414 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-09T05:48:01.467 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-09T05:48:01.525 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T05:48:01.539 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T05:48:01.583 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T05:48:01.836 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-paste. 2023-10-09T05:48:01.849 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-09T05:48:01.911 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T05:48:01.911 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T05:48:01.926 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T05:48:01.969 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T05:48:02.273 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-09T05:48:02.286 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-09T05:48:02.324 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T05:48:02.331 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T05:48:02.338 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T05:48:02.382 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T05:48:02.643 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-09T05:48:02.658 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-09T05:48:02.692 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-09T05:48:02.744 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T05:48:02.757 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:02.801 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:03.004 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-webtest. 2023-10-09T05:48:03.018 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-09T05:48:03.062 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-09T05:48:03.130 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T05:48:03.143 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:03.188 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:03.699 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T05:48:03.714 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:03.757 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:04.225 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pecan. 2023-10-09T05:48:04.238 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-09T05:48:04.291 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T05:48:04.654 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-09T05:48:04.670 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-09T05:48:04.713 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T05:48:05.066 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-09T05:48:05.082 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:05.124 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:05.228 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T05:48:05.241 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:05.294 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:05.478 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr. 2023-10-09T05:48:05.492 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:05.536 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:06.031 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mon. 2023-10-09T05:48:06.046 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:06.089 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:07.168 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-osd. 2023-10-09T05:48:07.181 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:07.226 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:07.891 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph. 2023-10-09T05:48:07.906 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:07.949 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:08.243 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T05:48:08.257 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:08.301 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:08.722 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T05:48:08.736 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:08.779 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:09.299 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package cephadm. 2023-10-09T05:48:09.314 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:09.357 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:09.668 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T05:48:09.682 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T05:48:09.726 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T05:48:09.824 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph. 2023-10-09T05:48:09.838 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../47-ceph_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:09.882 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:10.063 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T05:48:10.077 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:10.121 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:10.185 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-fuse. 2023-10-09T05:48:10.199 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:10.243 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:10.466 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T05:48:10.479 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T05:48:10.525 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T05:48:10.663 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mds. 2023-10-09T05:48:10.677 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:10.721 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:10.793 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T05:48:10.806 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T05:48:10.851 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T05:48:11.137 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T05:48:11.151 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T05:48:11.195 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T05:48:11.250 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package cephadm. 2023-10-09T05:48:11.263 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../50-cephadm_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:11.308 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:11.498 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-routes. 2023-10-09T05:48:11.511 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T05:48:11.556 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T05:48:11.628 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-jinja2. 2023-10-09T05:48:11.641 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-09T05:48:11.686 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-09T05:48:11.884 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T05:48:11.897 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:11.941 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:12.022 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-09T05:48:12.038 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:12.081 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:12.426 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-09T05:48:12.439 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-09T05:48:12.484 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-09T05:48:12.728 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libjs-underscore. 2023-10-09T05:48:12.742 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-09T05:48:12.786 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T05:48:12.907 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T05:48:12.923 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T05:48:12.965 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T05:48:13.064 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-09T05:48:13.077 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-09T05:48:13.131 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T05:48:13.251 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T05:48:13.265 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T05:48:13.309 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T05:48:13.467 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-routes. 2023-10-09T05:48:13.480 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-09T05:48:13.525 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-09T05:48:13.629 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T05:48:13.643 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T05:48:13.687 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T05:48:13.845 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-09T05:48:13.858 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:13.894 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:15.012 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-decorator. 2023-10-09T05:48:15.021 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T05:48:15.027 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-09T05:48:15.034 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T05:48:15.070 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T05:48:15.071 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:15.373 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-09T05:48:15.387 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-09T05:48:15.431 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T05:48:15.616 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T05:48:15.629 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T05:48:15.674 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T05:48:15.742 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-scipy. 2023-10-09T05:48:15.758 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-09T05:48:15.801 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-09T05:48:16.010 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T05:48:16.026 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T05:48:16.068 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:16.743 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T05:48:16.758 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:16.801 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:17.152 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-09T05:48:17.167 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-09T05:48:17.209 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:17.764 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-joblib. 2023-10-09T05:48:17.776 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-09T05:48:17.813 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-09T05:48:17.888 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T05:48:17.903 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T05:48:17.962 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T05:48:18.159 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-sklearn. 2023-10-09T05:48:18.174 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-09T05:48:18.217 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:18.265 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T05:48:18.279 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T05:48:18.323 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T05:48:18.634 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T05:48:18.650 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T05:48:18.692 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T05:48:18.959 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-09T05:48:18.973 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:18.995 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T05:48:19.011 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T05:48:19.017 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:19.053 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T05:48:19.365 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T05:48:19.379 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T05:48:19.440 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T05:48:19.743 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T05:48:19.758 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T05:48:19.792 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T05:48:20.104 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T05:48:20.119 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T05:48:20.137 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-cachetools. 2023-10-09T05:48:20.151 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-09T05:48:20.162 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T05:48:20.196 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-09T05:48:20.507 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-09T05:48:20.521 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-09T05:48:20.565 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T05:48:20.882 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T05:48:20.896 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T05:48:20.910 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-09T05:48:20.926 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-09T05:48:20.939 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T05:48:20.968 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T05:48:21.251 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T05:48:21.266 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T05:48:21.279 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-rsa. 2023-10-09T05:48:21.295 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-09T05:48:21.309 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T05:48:21.337 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-09T05:48:21.658 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-google-auth. 2023-10-09T05:48:21.662 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T05:48:21.674 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-09T05:48:21.676 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T05:48:21.716 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-09T05:48:21.720 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T05:48:22.019 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-blinker. 2023-10-09T05:48:22.035 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-09T05:48:22.077 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T05:48:22.316 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T05:48:22.331 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:22.356 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-09T05:48:22.370 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-09T05:48:22.374 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:22.405 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T05:48:22.677 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T05:48:22.691 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T05:48:22.709 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-09T05:48:22.723 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-09T05:48:22.735 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T05:48:22.759 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T05:48:23.020 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-websocket. 2023-10-09T05:48:23.035 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-09T05:48:23.078 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T05:48:23.088 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T05:48:23.103 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T05:48:23.146 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:23.390 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-09T05:48:23.403 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-09T05:48:23.415 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package jq. 2023-10-09T05:48:23.431 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T05:48:23.448 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T05:48:23.473 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:23.726 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package socat. 2023-10-09T05:48:23.740 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T05:48:23.784 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T05:48:24.036 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-09T05:48:24.051 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-48-gaeb2e425-1focal_all.deb ... 2023-10-09T05:48:24.095 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:24.104 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T05:48:24.119 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T05:48:24.162 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T05:48:24.397 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-09T05:48:24.411 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-09T05:48:24.455 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-09T05:48:24.581 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package ceph-test. 2023-10-09T05:48:24.590 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:24.631 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:24.915 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-09T05:48:24.924 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T05:48:25.117 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:25.395 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package jq. 2023-10-09T05:48:25.410 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-09T05:48:25.452 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:25.706 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package socat. 2023-10-09T05:48:25.720 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-09T05:48:25.764 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-09T05:48:26.084 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package xmlstarlet. 2023-10-09T05:48:26.097 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-09T05:48:26.142 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-09T05:48:26.565 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package ceph-test. 2023-10-09T05:48:26.580 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../81-ceph-test_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:26.623 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:28.910 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T05:48:28.924 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:28.968 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:29.371 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T05:48:29.385 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T05:48:29.445 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T05:48:29.828 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T05:48:29.842 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T05:48:29.877 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T05:48:30.172 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-attr. 2023-10-09T05:48:30.187 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T05:48:30.230 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T05:48:30.570 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T05:48:30.584 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T05:48:30.620 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T05:48:30.861 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-09T05:48:30.875 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:30.919 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:30.930 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T05:48:30.945 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T05:48:30.989 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T05:48:31.323 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package nvme-cli. 2023-10-09T05:48:31.333 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T05:48:31.337 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-09T05:48:31.347 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T05:48:31.395 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T05:48:31.398 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T05:48:31.836 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-09T05:48:31.843 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T05:48:31.851 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-09T05:48:31.852 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T05:48:31.893 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T05:48:31.894 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T05:48:32.195 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T05:48:32.197 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-attr. 2023-10-09T05:48:32.204 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T05:48:32.211 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-09T05:48:32.254 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T05:48:32.255 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-09T05:48:32.590 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T05:48:32.591 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-webencodings. 2023-10-09T05:48:32.599 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T05:48:32.605 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-09T05:48:32.649 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T05:48:32.657 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T05:48:32.944 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-html5lib. 2023-10-09T05:48:32.952 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T05:48:32.958 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-09T05:48:32.960 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T05:48:33.002 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T05:48:33.002 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-09T05:48:33.347 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-py. 2023-10-09T05:48:33.363 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T05:48:33.363 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-09T05:48:33.379 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-09T05:48:33.404 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T05:48:33.422 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T05:48:33.707 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T05:48:33.723 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T05:48:33.765 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T05:48:33.884 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-packaging. 2023-10-09T05:48:33.898 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-09T05:48:33.941 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-09T05:48:34.076 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T05:48:34.091 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T05:48:34.134 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T05:48:34.229 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pastescript. 2023-10-09T05:48:34.242 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-09T05:48:34.287 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-09T05:48:34.421 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T05:48:34.434 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T05:48:34.478 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T05:48:34.607 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pluggy. 2023-10-09T05:48:34.621 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-09T05:48:34.665 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-09T05:48:34.816 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T05:48:34.830 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T05:48:34.873 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T05:48:34.968 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-psutil. 2023-10-09T05:48:34.983 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-09T05:48:35.026 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T05:48:35.277 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package radosgw. 2023-10-09T05:48:35.293 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:35.335 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:35.380 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-py. 2023-10-09T05:48:35.394 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-09T05:48:35.438 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T05:48:35.758 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-09T05:48:35.772 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-09T05:48:35.849 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T05:48:36.152 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-09T05:48:36.167 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-09T05:48:36.210 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T05:48:36.513 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-pytest. 2023-10-09T05:48:36.528 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-09T05:48:36.555 INFO:teuthology.orchestra.run.smithi125.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T05:48:36.571 INFO:teuthology.orchestra.run.smithi125.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:36.572 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-09T05:48:36.613 INFO:teuthology.orchestra.run.smithi125.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:36.908 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package python3-simplejson. 2023-10-09T05:48:36.917 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-09T05:48:36.921 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T05:48:36.958 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T05:48:37.217 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T05:48:37.335 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T05:48:37.378 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package radosgw. 2023-10-09T05:48:37.387 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../98-radosgw_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:37.428 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:37.461 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T05:48:37.662 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:38.281 INFO:teuthology.orchestra.run.smithi125.stdout:Adding system user cephadm....done 2023-10-09T05:48:38.437 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T05:48:38.606 INFO:teuthology.orchestra.run.smithi155.stdout:Selecting previously unselected package rbd-fuse. 2023-10-09T05:48:38.615 INFO:teuthology.orchestra.run.smithi155.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-48-gaeb2e425-1focal_amd64.deb ... 2023-10-09T05:48:38.656 INFO:teuthology.orchestra.run.smithi155.stdout:Unpacking rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:38.779 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T05:48:38.962 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-09T05:48:39.074 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T05:48:39.200 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T05:48:39.253 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-09T05:48:39.379 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-09T05:48:39.507 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-09T05:48:39.509 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T05:48:39.708 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:39.919 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T05:48:40.204 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T05:48:40.345 INFO:teuthology.orchestra.run.smithi155.stdout:Adding system user cephadm....done 2023-10-09T05:48:40.489 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T05:48:40.497 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-09T05:48:40.781 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T05:48:40.849 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-09T05:48:41.066 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:41.142 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-09T05:48:41.268 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-09T05:48:41.350 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:41.476 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T05:48:41.603 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-09T05:48:41.760 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T05:48:41.997 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-09T05:48:42.045 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T05:48:42.290 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-09T05:48:42.420 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T05:48:42.566 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-09T05:48:42.697 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T05:48:42.842 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-09T05:48:43.065 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T05:48:43.126 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-ceph-argparse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:43.349 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:43.419 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:43.466 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T05:48:43.545 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-09T05:48:43.810 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T05:48:43.830 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-09T05:48:44.102 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T05:48:44.131 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-09T05:48:44.395 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T05:48:44.507 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-09T05:48:44.521 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T05:48:44.775 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-09T05:48:44.822 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T05:48:45.109 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-09T05:48:45.215 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T05:48:45.394 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libjaeger (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:45.542 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-09T05:48:45.567 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T05:48:45.880 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-09T05:48:45.918 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T05:48:46.044 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:46.156 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-09T05:48:46.379 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T05:48:46.457 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-09T05:48:46.583 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-09T05:48:46.722 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T05:48:46.856 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T05:48:46.867 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-09T05:48:46.982 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T05:48:47.253 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-09T05:48:47.267 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T05:48:47.585 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T05:48:47.612 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-09T05:48:47.712 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T05:48:47.981 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-09T05:48:48.038 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T05:48:48.107 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:48.323 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T05:48:48.417 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-09T05:48:48.615 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T05:48:48.733 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T05:48:48.753 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-09T05:48:48.879 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-09T05:48:49.005 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-09T05:48:49.009 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T05:48:49.289 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-09T05:48:49.302 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T05:48:49.603 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T05:48:49.607 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-09T05:48:49.733 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-09T05:48:50.016 INFO:teuthology.orchestra.run.smithi125.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T05:48:50.043 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-09T05:48:50.264 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T05:48:50.336 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-09T05:48:50.390 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T05:48:50.718 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T05:48:50.729 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-09T05:48:50.956 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-09T05:48:51.068 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T05:48:51.234 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T05:48:51.240 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-09T05:48:51.524 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-09T05:48:51.817 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-09T05:48:51.870 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T05:48:52.187 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T05:48:52.194 INFO:teuthology.orchestra.run.smithi155.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-09T05:48:52.462 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-09T05:48:52.499 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:52.588 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-09T05:48:52.626 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T05:48:52.916 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-09T05:48:53.278 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-09T05:48:53.404 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-09T05:48:53.530 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-09T05:48:53.864 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-09T05:48:54.090 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:54.157 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up librados2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:54.208 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:54.283 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-09T05:48:54.334 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:54.460 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T05:48:54.819 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T05:48:55.112 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T05:48:55.405 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T05:48:55.756 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up librgw2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:55.757 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:55.875 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T05:48:55.882 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libsqlite3-mod-ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:56.015 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:56.201 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:56.327 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T05:48:56.510 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-09T05:48:56.654 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:56.878 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-09T05:48:57.179 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-09T05:48:57.464 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-09T05:48:57.688 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:57.795 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libcephfs2 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:57.814 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T05:48:57.921 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-09T05:48:57.942 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:48:58.057 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T05:48:58.256 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libradosstriper1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:58.352 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T05:48:58.383 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-09T05:48:58.653 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:58.717 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-09T05:48:58.779 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:58.999 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T05:48:59.000 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T05:48:59.455 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:59.582 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T05:48:59.809 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up librbd1 (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:48:59.858 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T05:48:59.935 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-09T05:49:00.086 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-09T05:49:00.176 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T05:49:00.237 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-09T05:49:00.530 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-09T05:49:00.536 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:00.662 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:00.788 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:00.823 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr-modules-core (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:00.947 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T05:49:00.949 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:01.169 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T05:49:01.169 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-09T05:49:01.232 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T05:49:01.602 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:01.625 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up libcephfs-dev (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:01.728 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:01.751 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-09T05:49:02.021 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T05:49:02.053 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-09T05:49:02.362 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-09T05:49:02.707 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-rados (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:02.832 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-rbd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:02.958 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up rbd-fuse (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:02.983 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:03.084 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-09T05:49:03.377 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-09T05:49:03.528 INFO:teuthology.orchestra.run.smithi125.stdout:Adding group ceph....done 2023-10-09T05:49:03.756 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-rgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:03.882 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-cephfs (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:03.916 INFO:teuthology.orchestra.run.smithi125.stdout:Adding system user ceph....done 2023-10-09T05:49:04.092 INFO:teuthology.orchestra.run.smithi125.stdout:Setting system user ceph properties....done 2023-10-09T05:49:04.106 INFO:teuthology.orchestra.run.smithi125.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T05:49:04.217 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-09T05:49:04.254 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T05:49:04.575 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T05:49:05.046 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:05.131 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-common (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:05.171 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T05:49:05.442 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T05:49:05.686 INFO:teuthology.orchestra.run.smithi155.stdout:Adding group ceph....done 2023-10-09T05:49:06.070 INFO:teuthology.orchestra.run.smithi155.stdout:Adding system user ceph....done 2023-10-09T05:49:06.115 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:06.243 INFO:teuthology.orchestra.run.smithi155.stdout:Setting system user ceph properties....done 2023-10-09T05:49:06.265 INFO:teuthology.orchestra.run.smithi155.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-09T05:49:06.407 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-09T05:49:06.489 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T05:49:06.729 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-09T05:49:06.945 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:07.160 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T05:49:07.160 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T05:49:07.197 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-test (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:07.323 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up radosgw (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:07.595 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T05:49:07.595 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-09T05:49:07.621 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:07.845 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T05:49:07.845 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T05:49:08.249 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-base (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:08.306 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:08.629 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-09T05:49:08.795 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T05:49:08.796 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T05:49:09.085 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mds (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:09.250 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:09.310 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T05:49:09.311 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-09T05:49:09.384 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:09.554 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:09.777 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T05:49:09.777 INFO:teuthology.orchestra.run.smithi125.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T05:49:09.985 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T05:49:09.985 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-09T05:49:10.248 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:10.390 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:10.440 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-osd (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:10.516 INFO:teuthology.orchestra.run.smithi125.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:10.659 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T05:49:10.940 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T05:49:10.940 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-09T05:49:10.952 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T05:49:11.400 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr-k8sevents (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:11.526 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:11.660 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mon (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:11.886 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T05:49:11.886 INFO:teuthology.orchestra.run.smithi155.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-09T05:49:12.331 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr-cephadm (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:12.474 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:12.599 INFO:teuthology.orchestra.run.smithi155.stdout:Setting up ceph-mgr-dashboard (16.2.14-48-gaeb2e425-1focal) ... 2023-10-09T05:49:12.743 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-09T05:49:13.027 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-09T05:49:13.981 INFO:teuthology.orchestra.run.smithi125.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T05:49:14.430 INFO:teuthology.orchestra.run.smithi125.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T05:49:14.433 DEBUG:teuthology.parallel:result is None 2023-10-09T05:49:16.122 INFO:teuthology.orchestra.run.smithi155.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-09T05:49:16.586 INFO:teuthology.orchestra.run.smithi155.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-09T05:49:16.589 DEBUG:teuthology.parallel:result is None 2023-10-09T05:49:16.589 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:49:16.589 INFO:teuthology.packaging:ref: None 2023-10-09T05:49:16.589 INFO:teuthology.packaging:tag: None 2023-10-09T05:49:16.589 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:49:16.589 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:49:16.590 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T05:49:16.796 DEBUG:teuthology.orchestra.run.smithi125:> dpkg-query -W -f '${Version}' ceph 2023-10-09T05:49:16.819 INFO:teuthology.orchestra.run.smithi125.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T05:49:16.820 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T05:49:16.820 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T05:49:16.820 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:49:16.820 INFO:teuthology.packaging:ref: None 2023-10-09T05:49:16.820 INFO:teuthology.packaging:tag: None 2023-10-09T05:49:16.820 INFO:teuthology.packaging:branch: pacific 2023-10-09T05:49:16.820 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:49:16.821 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-09T05:49:17.023 DEBUG:teuthology.orchestra.run.smithi155:> dpkg-query -W -f '${Version}' ceph 2023-10-09T05:49:17.046 INFO:teuthology.orchestra.run.smithi155.stdout:16.2.14-48-gaeb2e425-1focal 2023-10-09T05:49:17.047 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-48-gaeb2e425-1focal 2023-10-09T05:49:17.047 INFO:teuthology.task.install:The correct ceph version 16.2.14-48-gaeb2e425-1focal is installed. 2023-10-09T05:49:17.047 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T05:49:17.047 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:49:17.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:49:17.063 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:49:17.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:49:17.105 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T05:49:17.105 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:49:17.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:49:17.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:49:17.177 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:49:17.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:49:17.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:49:17.252 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T05:49:17.252 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:49:17.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:49:17.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:49:17.325 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:49:17.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:49:17.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:49:17.400 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T05:49:17.401 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:49:17.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:49:17.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:49:17.473 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:49:17.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:49:17.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:49:17.543 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T05:49:17.616 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:49:17.617 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T05:49:17.617 INFO:tasks.cephadm:Cluster fsid is 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:49:17.617 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T05:49:17.618 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.125', 'mon.c': '[v2:172.21.15.125:3301,v1:172.21.15.125:6790]', 'mon.b': '172.21.15.155'} 2023-10-09T05:49:17.618 INFO:tasks.cephadm:First mon is mon.a on smithi125 2023-10-09T05:49:17.618 INFO:tasks.cephadm:First mgr is y 2023-10-09T05:49:17.618 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T05:49:17.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo hostname $(hostname -s) 2023-10-09T05:49:17.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo hostname $(hostname -s) 2023-10-09T05:49:17.650 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-09T05:49:17.650 DEBUG:teuthology.orchestra.run.smithi125:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:17.828 INFO:teuthology.orchestra.run.smithi125.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 05:49 /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:17.828 DEBUG:teuthology.orchestra.run.smithi155:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:17.985 INFO:teuthology.orchestra.run.smithi155.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 9 05:49 /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:17.986 DEBUG:teuthology.orchestra.run.smithi125:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:17.995 DEBUG:teuthology.orchestra.run.smithi155:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:49:18.013 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-09T05:49:18.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T05:49:18.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-09T05:49:18.308 INFO:teuthology.orchestra.run.smithi125.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:49:18.343 INFO:teuthology.orchestra.run.smithi155.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:50:34.699 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout: "repo_digests": [ 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T05:50:34.700 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T05:50:46.157 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T05:50:46.157 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph_version": "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)", 2023-10-09T05:50:46.157 INFO:teuthology.orchestra.run.smithi125.stdout: "image_id": "83c1f8bb3ad8bc51d249aef2da9bdf52b5282074d63d7c4f2c8c24d22f4df352", 2023-10-09T05:50:46.157 INFO:teuthology.orchestra.run.smithi125.stdout: "repo_digests": [ 2023-10-09T05:50:46.157 INFO:teuthology.orchestra.run.smithi125.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3" 2023-10-09T05:50:46.158 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T05:50:46.158 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T05:50:46.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo mkdir -p /etc/ceph 2023-10-09T05:50:46.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo mkdir -p /etc/ceph 2023-10-09T05:50:46.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 777 /etc/ceph 2023-10-09T05:50:46.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 777 /etc/ceph 2023-10-09T05:50:46.275 INFO:tasks.cephadm:Writing seed config... 2023-10-09T05:50:46.276 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T05:50:46.276 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T05:50:46.276 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T05:50:46.276 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T05:50:46.277 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T05:50:46.278 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:50:46.278 DEBUG:teuthology.orchestra.run.smithi125:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T05:50:46.304 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 95d0291c-6667-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T05:50:46.305 DEBUG:teuthology.orchestra.run.smithi125:mon.a> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T05:50:46.348 DEBUG:teuthology.orchestra.run.smithi125:mgr.y> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T05:50:46.391 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T05:50:46.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.125 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:50:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:-------------------------------------------------------------------------------- 2023-10-09T05:50:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '95d0291c-6667-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.125', '--skip-admin-label'] 2023-10-09T05:50:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying podman|docker is present... 2023-10-09T05:50:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying lvm2 is present... 2023-10-09T05:50:46.607 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying time synchronization is in place... 2023-10-09T05:50:46.612 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:50:46.619 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.625 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T05:50:46.631 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.637 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: masked 2023-10-09T05:50:46.643 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.647 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T05:50:46.653 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.659 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: enabled 2023-10-09T05:50:46.663 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: active 2023-10-09T05:50:46.664 INFO:teuthology.orchestra.run.smithi125.stderr:Unit ntp.service is enabled and running 2023-10-09T05:50:46.664 INFO:teuthology.orchestra.run.smithi125.stderr:Repeating the final host check... 2023-10-09T05:50:46.664 INFO:teuthology.orchestra.run.smithi125.stderr:docker (/usr/bin/docker) is present 2023-10-09T05:50:46.664 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl is present 2023-10-09T05:50:46.665 INFO:teuthology.orchestra.run.smithi125.stderr:lvcreate is present 2023-10-09T05:50:46.669 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:50:46.676 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.683 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-09T05:50:46.687 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.694 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: masked 2023-10-09T05:50:46.699 INFO:journalctl@ceph.mgr.y.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:50:46.700 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.707 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-09T05:50:46.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:50:46.714 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: inactive 2023-10-09T05:50:46.720 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: enabled 2023-10-09T05:50:46.726 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: active 2023-10-09T05:50:46.727 INFO:teuthology.orchestra.run.smithi125.stderr:Unit ntp.service is enabled and running 2023-10-09T05:50:46.727 INFO:teuthology.orchestra.run.smithi125.stderr:Host looks OK 2023-10-09T05:50:46.727 INFO:teuthology.orchestra.run.smithi125.stderr:Cluster fsid: 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Acquiring lock 140455297871824 on /run/cephadm/95d0291c-6667-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:50:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Lock 140455297871824 acquired on /run/cephadm/95d0291c-6667-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:50:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying IP 172.21.15.125 port 3300 ... 2023-10-09T05:50:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying IP 172.21.15.125 port 6789 ... 2023-10-09T05:50:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Base mon IP(s) is [172.21.15.125:3300, 172.21.15.125:6789], mon addrv is [v2:172.21.15.125:3300,v1:172.21.15.125:6789] 2023-10-09T05:50:46.732 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-10-09T05:50:46.732 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.125 metric 100 2023-10-09T05:50:46.733 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-09T05:50:46.733 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.125 2023-10-09T05:50:46.733 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.125 metric 100 2023-10-09T05:50:46.736 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T05:50:46.736 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-09T05:50:46.736 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1795sec pref medium 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d739/64 scope link 2023-10-09T05:50:46.740 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-09T05:50:46.741 INFO:teuthology.orchestra.run.smithi125.stderr:Mon IP `172.21.15.125` is in CIDR network `172.21.0.0/20` 2023-10-09T05:50:46.741 INFO:teuthology.orchestra.run.smithi125.stderr:Mon IP `172.21.15.125` is in CIDR network `172.21.0.0/20` 2023-10-09T05:50:46.741 INFO:teuthology.orchestra.run.smithi125.stderr:Mon IP `172.21.15.125` is in CIDR network `172.21.15.254/32` 2023-10-09T05:50:46.742 INFO:teuthology.orchestra.run.smithi125.stderr:Mon IP `172.21.15.125` is in CIDR network `172.21.15.254/32` 2023-10-09T05:50:46.742 INFO:teuthology.orchestra.run.smithi125.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-10-09T05:50:46.742 INFO:teuthology.orchestra.run.smithi125.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T05:50:46.743 INFO:teuthology.orchestra.run.smithi125.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-09T05:50:46.947 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-10-09T05:50:46.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/docker: Digest: sha256:4f464afc8f1cc01018a8c87b9dd7b2223f549df43bea1784673b845fee6e24c3 2023-10-09T05:50:46.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T05:50:47.002 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-10-09T05:50:47.902 INFO:teuthology.orchestra.run.smithi125.stderr:ceph: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T05:50:48.435 INFO:teuthology.orchestra.run.smithi125.stderr:Ceph version: ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable) 2023-10-09T05:50:48.435 INFO:teuthology.orchestra.run.smithi125.stderr:Extracting ceph user uid/gid from container image... 2023-10-09T05:50:49.260 INFO:teuthology.orchestra.run.smithi125.stderr:stat: 167 167 2023-10-09T05:50:49.906 INFO:teuthology.orchestra.run.smithi125.stderr:Creating initial keys... 2023-10-09T05:50:50.663 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-authtool: AQC6lCNlegN6JxAA6o/EQjYuHNsXQucn3RKGCQ== 2023-10-09T05:50:52.099 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-authtool: AQC8lCNlmPXlBRAAPCGT1VqCeVHoo/poTOv5+Q== 2023-10-09T05:50:53.527 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-authtool: AQC9lCNll+FkHxAAFiLQwECZvbgO5+FZmkbyRQ== 2023-10-09T05:50:54.096 INFO:teuthology.orchestra.run.smithi125.stderr:Creating initial monmap... 2023-10-09T05:50:54.895 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:50:54.895 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:54.895 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:50:55.499 INFO:teuthology.orchestra.run.smithi125.stderr:monmaptool for a [v2:172.21.15.125:3300,v1:172.21.15.125:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:50:55.499 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/monmaptool: set fsid to 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:55.500 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:50:55.500 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:50:55.500 INFO:teuthology.orchestra.run.smithi125.stderr:Creating mon... 2023-10-09T05:50:56.277 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:50:56.277 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 1 imported monmap: 2023-10-09T05:50:56.278 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-09T05:50:56.278 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:56.278 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: last_changed 2023-10-09T05:50:54.893889+0000 2023-10-09T05:50:56.278 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: created 2023-10-09T05:50:54.893889+0000 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] mon.a 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 0 /usr/bin/ceph-mon: set fsid to 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.301 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: DB SUMMARY 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.create_if_missing: 1 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:50:56.302 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.env: 0x5585e2fc1080 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.info_log: 0x5585e4035aa0 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:50:56.303 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:50:56.304 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.write_buffer_manager: 0x5585e403d890 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:50:56.305 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.row_cache: None 2023-10-09T05:50:56.306 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:50:56.307 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:50:56.308 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:50:56.309 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.273+0000 7ff0ca2cc880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.310 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.merge_operator: 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5585e3f4bd20) 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-09T05:50:56.311 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_cache: 0x5585e3f82d10 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-09T05:50:56.312 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-09T05:50:56.313 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:50:56.314 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:50:56.315 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:50:56.316 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:50:56.317 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:50:56.318 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:50:56.319 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.277+0000 7ff0ca2cc880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.281+0000 7ff0ca2cc880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.281+0000 7ff0ca2cc880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.281+0000 7ff0ca2cc880 4 rocksdb: DB pointer 0x5585e404b800 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.281+0000 7ff0b3154700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:50:56.320 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.281+0000 7ff0b3154700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:50:56.321 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:50:56.322 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:50:56.323 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:50:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.285+0000 7ff0ca2cc880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.285+0000 7ff0ca2cc880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-09T05:50:56.325 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph-mon: debug 2023-10-09T05:50:56.285+0000 7ff0ca2cc880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-09T05:50:56.929 INFO:teuthology.orchestra.run.smithi125.stderr:create mon.a on 2023-10-09T05:50:56.955 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T05:50:57.171 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-09T05:50:57.380 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:50:57.380 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:50:57.818 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to reset failed state of unit ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T05:50:57.825 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Created symlink /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target.wants/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:50:58.037 INFO:teuthology.orchestra.run.smithi125.stderr:firewalld does not appear to be present 2023-10-09T05:50:58.037 INFO:teuthology.orchestra.run.smithi125.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T05:50:58.037 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mon to start... 2023-10-09T05:50:58.038 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mon... 2023-10-09T05:50:58.088 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:50:58 smithi125 systemd[1]: Started Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:50:59.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:50:59 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.273186+0000 mon.a (mon.0) 0 : [INF] mkfs 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:59.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:50:59 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.258971+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:50:59.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: cluster: 2023-10-09T05:50:59.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: id: 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:59.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-09T05:50:59.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:50:59.480 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: services: 2023-10-09T05:50:59.480 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.208778s) 2023-10-09T05:50:59.480 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: data: 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: pgs: 2023-10-09T05:50:59.481 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:00.069 INFO:teuthology.orchestra.run.smithi125.stderr:mon is available 2023-10-09T05:51:00.070 INFO:teuthology.orchestra.run.smithi125.stderr:Assimilating anything we can from ceph.conf... 2023-10-09T05:51:00.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.268477+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:51:00.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.270578+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T05:51:00.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.275257+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-09T05:51:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.278850+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:51:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: cluster 2023-10-09T05:50:59.279319+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T05:51:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:00 smithi125 bash[26988]: audit 2023-10-09T05:50:59.477164+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.125:0/2734674024' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T05:51:01.084 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:01.084 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [global] 2023-10-09T05:51:01.084 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: fsid = 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:01.085 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T05:51:01.086 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.125:3300,v1:172.21.15.125:6789] 2023-10-09T05:51:01.086 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T05:51:01.086 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T05:51:01.086 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T05:51:01.086 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [mgr] 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [osd] 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T05:51:01.087 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T05:51:01.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:01 smithi125 bash[26988]: audit 2023-10-09T05:51:01.076571+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.125:0/2620012420' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:51:01.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:01 smithi125 bash[26988]: audit 2023-10-09T05:51:01.082730+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.125:0/2620012420' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T05:51:01.691 INFO:teuthology.orchestra.run.smithi125.stderr:Generating new minimal ceph.conf... 2023-10-09T05:51:03.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:02 smithi125 bash[26988]: audit 2023-10-09T05:51:02.710568+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.125:0/586137515' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:03.280 INFO:teuthology.orchestra.run.smithi125.stderr:Restarting the monitor... 2023-10-09T05:51:03.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:03 smithi125 systemd[1]: Stopping Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:03.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:03 smithi125 bash[27372]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon.a 2023-10-09T05:51:03.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:03 smithi125 bash[26988]: debug 2023-10-09T05:51:03.457+0000 7fc09f029700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T05:51:03.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:03 smithi125 bash[26988]: debug 2023-10-09T05:51:03.457+0000 7fc09f029700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T05:51:04.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:03 smithi125 bash[27410]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T05:51:04.062 INFO:teuthology.orchestra.run.smithi125.stderr:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-10-09T05:51:04.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:04 smithi125 bash[27472]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T05:51:04.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:04 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:51:04.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:04 smithi125 systemd[1]: Stopped Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:04.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:04 smithi125 systemd[1]: Started Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.161+0000 7f79ed787880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.161+0000 7f79ed787880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.161+0000 7f79ed787880 0 pidfile_write: ignore empty --pid-file 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 0 load: jerasure load: lrc load: isa 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:51:05.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: DB SUMMARY 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86737 ; 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:51:05.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.env: 0x55cc723c1080 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.info_log: 0x55cc7415fa80 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:51:05.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:51:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:51:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:51:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:51:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:51:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:51:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:51:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:51:05.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.write_buffer_manager: 0x55cc74e5c2d0 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:51:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:51:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.row_cache: None 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:51:05.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:51:05.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:51:05.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:51:05.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.merge_operator: 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:51:05.397 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cc7410dd18) 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cache_index_and_filter_blocks: 1 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: pin_top_level_index_and_filter: 1 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: index_type: 0 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: data_block_index_type: 0 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: index_shortening: 1 2023-10-09T05:51:05.398 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: hash_index_allow_collision: 1 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: checksum: 1 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: no_block_cache: 0 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_cache: 0x55cc74144f10 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_cache_name: BinnedLRUCache 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_cache_options: 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: capacity : 536870912 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: num_shard_bits : 4 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: strict_capacity_limit : 0 2023-10-09T05:51:05.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: high_pri_pool_ratio: 0.000 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_cache_compressed: (nil) 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: persistent_cache: (nil) 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_size: 4096 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_size_deviation: 10 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_restart_interval: 16 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: index_block_restart_interval: 1 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: metadata_block_size: 4096 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: partition_filters: 0 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: use_delta_encoding: 1 2023-10-09T05:51:05.400 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: whole_key_filtering: 1 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: verify_compression: 0 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: read_amp_bytes_per_bit: 0 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: format_version: 2 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: enable_index_compression: 1 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: block_align: 0 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:51:05.401 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:51:05.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:51:05.403 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:51:05.404 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:51:05.405 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:51:05.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:51:05.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830665174360, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.169+0000 7f79ed787880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-09T05:51:05.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.173+0000 7f79ed787880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.173+0000 7f79ed787880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830665176221, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83337, "table_properties": {"data_size": 81610, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75931, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696830665, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.173+0000 7f79ed787880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.173+0000 7f79ed787880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830665176816, "job": 1, "event": "recovery_finished"} 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.173+0000 7f79ed787880 4 rocksdb: DB pointer 0x55cc7420d800 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79d458e700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79d458e700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** DB Stats ** 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:51:05.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** Compaction Stats [default] ** 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: L0 2/0 83.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Sum 2/0 83.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** Compaction Stats [default] ** 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:51:05.411 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Cumulative compaction: 0.00 GB write, 13.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Interval compaction: 0.00 GB write, 13.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** Compaction Stats [default] ** 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: L0 2/0 83.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Sum 2/0 83.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:51:05.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** Compaction Stats [default] ** 2023-10-09T05:51:05.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:05.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:05.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 72.4 0.00 0.00 1 0.001 0 0 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Cumulative compaction: 0.00 GB write, 13.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:05.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] at bind addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 1 mon.a@-1(???) e1 preinit fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).mds e1 new map 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).mds e1 print_map 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: e1 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: legacy client fscid: -1 2023-10-09T05:51:05.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: No filesystems configured 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: audit 2023-10-09T05:51:02.710568+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.125:0/586137515' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: debug 2023-10-09T05:51:05.177+0000 7f79ed787880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cluster 2023-10-09T05:51:05.203358+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cluster 2023-10-09T05:51:05.203482+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0]} removed_ranks: {} 2023-10-09T05:51:05.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cluster 2023-10-09T05:51:05.204541+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-09T05:51:05.417 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cluster 2023-10-09T05:51:05.204652+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:51:05.417 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:05 smithi125 bash[27513]: cluster 2023-10-09T05:51:05.205518+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-09T05:51:06.072 INFO:teuthology.orchestra.run.smithi125.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-09T05:51:06.072 INFO:teuthology.orchestra.run.smithi125.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:51:06.073 INFO:teuthology.orchestra.run.smithi125.stderr:Creating mgr... 2023-10-09T05:51:06.073 INFO:teuthology.orchestra.run.smithi125.stderr:Verifying port 9283 ... 2023-10-09T05:51:06.303 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Failed to reset failed state of unit ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T05:51:06.311 INFO:teuthology.orchestra.run.smithi125.stderr:systemctl: Created symlink /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7.target.wants/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:firewalld does not appear to be present 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:Not possible to enable service . firewalld.service is not available 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:firewalld does not appear to be present 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mgr to start... 2023-10-09T05:51:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mgr... 2023-10-09T05:51:06.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:06 smithi125 bash[27513]: audit 2023-10-09T05:51:05.434054+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.125:0/826763344' entity='client.admin' 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsid": "95d0291c-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:51:08.029 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 0 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "a" 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:51:08.030 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:51:08.031 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:51:08.034 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:51:08.035 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "available": false, 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:51:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:50:59.273598+0000", 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:51:08.037 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:08.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:08 smithi125 bash[27513]: audit 2023-10-09T05:51:08.025339+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.125:0/3703031688' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:08.268 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:08 smithi125 bash[27779]: debug 2023-10-09T05:51:08.001+0000 7fdb69e5f680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:51:08.634 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:08 smithi125 bash[27779]: debug 2023-10-09T05:51:08.481+0000 7fdb69e5f680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:51:08.712 INFO:teuthology.orchestra.run.smithi125.stderr:mgr not available, waiting (1/15)... 2023-10-09T05:51:09.817 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:09 smithi125 bash[27779]: debug 2023-10-09T05:51:09.473+0000 7fdb69e5f680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:51:09.817 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:09 smithi125 bash[27779]: debug 2023-10-09T05:51:09.569+0000 7fdb69e5f680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:51:10.134 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:09 smithi125 bash[27779]: debug 2023-10-09T05:51:09.813+0000 7fdb69e5f680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:51:11.127 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:10 smithi125 bash[27779]: debug 2023-10-09T05:51:10.961+0000 7fdb69e5f680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:51:11.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27779]: debug 2023-10-09T05:51:11.125+0000 7fdb69e5f680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:11.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27779]: debug 2023-10-09T05:51:11.225+0000 7fdb69e5f680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:11.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27779]: debug 2023-10-09T05:51:11.341+0000 7fdb69e5f680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:51:11.773 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:11.773 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:11.773 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsid": "95d0291c-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:11.773 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 0 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:51:11.774 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "a" 2023-10-09T05:51:11.775 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:11.775 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-09T05:51:11.775 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:51:11.776 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:51:11.777 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:51:11.777 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:51:11.780 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "available": false, 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:11.782 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:50:59.273598+0000", 2023-10-09T05:51:11.783 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:11.783 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:11.783 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:51:11.783 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:11.816 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27779]: debug 2023-10-09T05:51:11.561+0000 7fdb69e5f680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:51:12.086 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27513]: audit 2023-10-09T05:51:11.770843+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.125:0/791943125' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:12.086 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:11 smithi125 bash[27779]: debug 2023-10-09T05:51:11.953+0000 7fdb69e5f680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:51:12.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:12 smithi125 bash[27779]: debug 2023-10-09T05:51:12.081+0000 7fdb69e5f680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:12.413 INFO:teuthology.orchestra.run.smithi125.stderr:mgr not available, waiting (2/15)... 2023-10-09T05:51:13.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:12 smithi125 bash[27779]: debug 2023-10-09T05:51:12.957+0000 7fdb69e5f680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:51:13.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:13 smithi125 bash[27779]: debug 2023-10-09T05:51:13.061+0000 7fdb69e5f680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:51:13.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:13 smithi125 bash[27779]: debug 2023-10-09T05:51:13.177+0000 7fdb69e5f680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:51:13.685 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:13 smithi125 bash[27779]: debug 2023-10-09T05:51:13.397+0000 7fdb69e5f680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:51:14.019 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:13 smithi125 bash[27779]: debug 2023-10-09T05:51:13.681+0000 7fdb69e5f680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:51:14.020 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:13 smithi125 bash[27779]: debug 2023-10-09T05:51:13.845+0000 7fdb69e5f680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:51:14.336 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27779]: debug 2023-10-09T05:51:14.017+0000 7fdb69e5f680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:14.336 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27779]: debug 2023-10-09T05:51:14.225+0000 7fdb69e5f680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:51:14.634 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27779]: debug 2023-10-09T05:51:14.333+0000 7fdb69e5f680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: cluster 2023-10-09T05:51:14.356778+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: cluster 2023-10-09T05:51:14.363486+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00695601s) 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.365004+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.365312+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.365572+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.365820+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:51:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.366390+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: cluster 2023-10-09T05:51:14.371035+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.379543+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.388960+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.393141+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.397993+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' 2023-10-09T05:51:14.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:14 smithi125 bash[27513]: audit 2023-10-09T05:51:14.400707+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.125:0/4150236347' entity='mgr.y' 2023-10-09T05:51:15.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:15.479 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:15.484 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsid": "95d0291c-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "health": { 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "checks": {}, 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mutes": [] 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum": [ 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 0 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:15.485 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "a" 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "monmap": { 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osdmap": { 2023-10-09T05:51:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgmap": { 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-09T05:51:15.487 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "fsmap": { 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:15.488 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modules": [ 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "iostat", 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "nfs", 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "restful" 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ], 2023-10-09T05:51:15.489 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "servicemap": { 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "modified": "2023-10-09T05:50:59.273598+0000", 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "services": {} 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: }, 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-09T05:51:15.490 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:16.080 INFO:teuthology.orchestra.run.smithi125.stderr:mgr is available 2023-10-09T05:51:16.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:16 smithi125 bash[27513]: cluster 2023-10-09T05:51:15.367983+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01146s) 2023-10-09T05:51:16.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:16 smithi125 bash[27513]: audit 2023-10-09T05:51:15.477546+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.125:0/3303303737' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:17.295 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:17.295 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [global] 2023-10-09T05:51:17.295 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: fsid = 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:17.295 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-09T05:51:17.296 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-09T05:51:17.296 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-09T05:51:17.296 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-09T05:51:17.296 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-09T05:51:17.297 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:17.297 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [mgr] 2023-10-09T05:51:17.297 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-09T05:51:17.297 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 2023-10-09T05:51:17.297 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: [osd] 2023-10-09T05:51:17.298 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-09T05:51:17.298 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-09T05:51:17.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:17 smithi125 bash[27513]: cluster 2023-10-09T05:51:16.366983+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-10-09T05:51:17.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:17 smithi125 bash[27513]: audit 2023-10-09T05:51:17.292467+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.125:0/1198941568' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:51:17.929 INFO:teuthology.orchestra.run.smithi125.stderr:Enabling cephadm module... 2023-10-09T05:51:19.594 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:19 smithi125 bash[27513]: audit 2023-10-09T05:51:19.239060+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.125:0/4294252377' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T05:51:19.595 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:19 smithi125 bash[27779]: ignoring --setuser ceph since I am not root 2023-10-09T05:51:19.595 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:19 smithi125 bash[27779]: ignoring --setgroup ceph since I am not root 2023-10-09T05:51:19.595 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:19 smithi125 bash[27779]: debug 2023-10-09T05:51:19.489+0000 7f7f25e5d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:51:19.595 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:19 smithi125 bash[27779]: debug 2023-10-09T05:51:19.589+0000 7f7f25e5d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:51:20.302 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:20 smithi125 bash[27779]: debug 2023-10-09T05:51:20.037+0000 7f7f25e5d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:51:20.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:20 smithi125 bash[27513]: audit 2023-10-09T05:51:19.295603+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.125:0/4294252377' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T05:51:20.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:20 smithi125 bash[27513]: cluster 2023-10-09T05:51:19.295696+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 4s) 2023-10-09T05:51:21.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:20 smithi125 bash[27779]: debug 2023-10-09T05:51:20.985+0000 7f7f25e5d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:51:21.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:21 smithi125 bash[27779]: debug 2023-10-09T05:51:21.085+0000 7f7f25e5d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T05:51:21.409 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:21.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:21 smithi125 bash[27513]: audit 2023-10-09T05:51:21.407436+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.125:0/2400541283' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:51:21.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:21 smithi125 bash[27779]: debug 2023-10-09T05:51:21.337+0000 7f7f25e5d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:51:22.062 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for the mgr to restart... 2023-10-09T05:51:22.063 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mgr epoch 5... 2023-10-09T05:51:22.740 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:22 smithi125 bash[27779]: debug 2023-10-09T05:51:22.477+0000 7f7f25e5d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:51:22.740 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:22 smithi125 bash[27779]: debug 2023-10-09T05:51:22.645+0000 7f7f25e5d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:23.077 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:22 smithi125 bash[27779]: debug 2023-10-09T05:51:22.745+0000 7f7f25e5d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:23.077 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:22 smithi125 bash[27779]: debug 2023-10-09T05:51:22.857+0000 7f7f25e5d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:51:23.884 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:23 smithi125 bash[27779]: debug 2023-10-09T05:51:23.073+0000 7f7f25e5d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:51:23.884 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:23 smithi125 bash[27779]: debug 2023-10-09T05:51:23.437+0000 7f7f25e5d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:51:23.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:23 smithi125 bash[27779]: debug 2023-10-09T05:51:23.557+0000 7f7f25e5d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:24.861 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:24 smithi125 bash[27779]: debug 2023-10-09T05:51:24.417+0000 7f7f25e5d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:51:24.861 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:24 smithi125 bash[27779]: debug 2023-10-09T05:51:24.521+0000 7f7f25e5d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:51:24.861 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:24 smithi125 bash[27779]: debug 2023-10-09T05:51:24.637+0000 7f7f25e5d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:51:25.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:24 smithi125 bash[27779]: debug 2023-10-09T05:51:24.857+0000 7f7f25e5d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:51:25.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:25 smithi125 bash[27779]: debug 2023-10-09T05:51:25.137+0000 7f7f25e5d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:51:25.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:25 smithi125 bash[27779]: debug 2023-10-09T05:51:25.301+0000 7f7f25e5d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:51:25.794 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:25 smithi125 bash[27779]: debug 2023-10-09T05:51:25.469+0000 7f7f25e5d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:25.794 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:25 smithi125 bash[27779]: debug 2023-10-09T05:51:25.685+0000 7f7f25e5d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:51:26.634 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:25 smithi125 bash[27779]: debug 2023-10-09T05:51:25.789+0000 7f7f25e5d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: cluster 2023-10-09T05:51:25.812715+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: cluster 2023-10-09T05:51:25.813161+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: cluster 2023-10-09T05:51:25.819594+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: cluster 2023-10-09T05:51:25.819710+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00671965s) 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.821541+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.822725+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.823811+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:51:26.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.824102+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.824383+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: cluster 2023-10-09T05:51:25.828378+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.849418+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.978165+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.980441+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.983163+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:25.985955+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:26.011478+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:51:26.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:26 smithi125 bash[27513]: audit 2023-10-09T05:51:26.016136+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:51:26.838 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:26.838 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-09T05:51:26.838 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T05:51:26.838 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:27.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:27 smithi125 bash[27513]: cephadm 2023-10-09T05:51:25.843638+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-09T05:51:27.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:27 smithi125 bash[27513]: cluster 2023-10-09T05:51:26.823496+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01049s) 2023-10-09T05:51:27.501 INFO:teuthology.orchestra.run.smithi125.stderr:mgr epoch 5 is available 2023-10-09T05:51:27.502 INFO:teuthology.orchestra.run.smithi125.stderr:Setting orchestrator backend to cephadm... 2023-10-09T05:51:28.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:28 smithi125 bash[27513]: audit 2023-10-09T05:51:26.822449+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:51:28.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:28 smithi125 bash[27513]: audit 2023-10-09T05:51:26.836507+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:51:29.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:29 smithi125 bash[27513]: cluster 2023-10-09T05:51:28.196248+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-09T05:51:29.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:29 smithi125 bash[27513]: audit 2023-10-09T05:51:28.879579+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:29.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:29 smithi125 bash[27513]: audit 2023-10-09T05:51:28.917224+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:30.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:30 smithi125 bash[27513]: audit 2023-10-09T05:51:28.872874+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:30.883 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: value unchanged 2023-10-09T05:51:31.487 INFO:teuthology.orchestra.run.smithi125.stderr:Generating ssh key... 2023-10-09T05:51:31.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:31 smithi125 bash[27513]: audit 2023-10-09T05:51:30.881161+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:33.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: Generating public/private rsa key pair. 2023-10-09T05:51:33.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: Your identification has been saved in /tmp/tmpwjj6vtd_/key. 2023-10-09T05:51:33.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: Your public key has been saved in /tmp/tmpwjj6vtd_/key.pub. 2023-10-09T05:51:33.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: The key fingerprint is: 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: SHA256:ifzD8Zh0jAzMa2fr84OxTtcbsFs4d5AZTHStl48w6rc ceph-95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: The key's randomart image is: 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: +---[RSA 3072]----+ 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | .o .. | 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | o o . .| 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | + o ..| 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | . = + o+...| 2023-10-09T05:51:33.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | = S +.+o o.| 2023-10-09T05:51:33.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | . *.B.= .. .| 2023-10-09T05:51:33.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | O=* = . | 2023-10-09T05:51:33.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | o+oo=.+ | 2023-10-09T05:51:33.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: | .ooooE. | 2023-10-09T05:51:33.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:33 smithi125 bash[27779]: +----[SHA256]-----+ 2023-10-09T05:51:34.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:34 smithi125 bash[27513]: audit 2023-10-09T05:51:32.878826+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:34.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:34 smithi125 bash[27513]: cephadm 2023-10-09T05:51:32.879261+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-09T05:51:34.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:34 smithi125 bash[27513]: audit 2023-10-09T05:51:33.166241+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:34.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:34 smithi125 bash[27513]: audit 2023-10-09T05:51:33.168739+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:34.943 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpJziRse2FgxMF5YKXEOkNOqKFmqy5fjTAMiz5WeRtbTjXkC/MKpfXAqvFT31VHqwGBIRZfgjvs+EzDxm+Fro+4G25JizzW362gBhXyd6Tlvqf2+Q9b7ikZtSTYrNOcVRAcfcmJBioRZlk0XLmv6LF/mk8aFB9VHnW6itY9HbCXDpQQaRXT/viNG2v/GzyXC+YIhZVxGCasxdRk629kL09ZBCDuWMMBvpZXGGqf7GG3MAVGhPk713QsJgnbtbQrNvEClVtn2FpFTXxej2tlr0xAinOB46mQuRaY1dnZB8sBIpL2kOBx+U+Dz6i8bHlr4CnPrjWTaM5OQ4bBQyXP11ZMroiLMwjryZuHsYrEbv7xo0AxuxV5haxocsh58OzUJr+PPobECmX65DkttA8W2MoSqUAQ4rEm0iFhYcvPon20j+D8MZonjuDspMxz0nI36sjyuLMZUS9VFL+/2Rr5Y5tA5FoHkTMvGPX0edFkPomkSlY5PoaDcjRFpePjrHr07U= ceph-95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:35.572 INFO:teuthology.orchestra.run.smithi125.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:51:35.572 INFO:teuthology.orchestra.run.smithi125.stderr:Adding key to root@localhost authorized_keys... 2023-10-09T05:51:35.573 INFO:teuthology.orchestra.run.smithi125.stderr:Adding host smithi125... 2023-10-09T05:51:36.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:36 smithi125 bash[27513]: audit 2023-10-09T05:51:34.941626+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:37.134 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:36 smithi125 bash[27779]: Warning: Permanently added '172.21.15.125' (ECDSA) to the list of known hosts. 2023-10-09T05:51:37.601 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: Added host 'smithi125' with addr '172.21.15.125' 2023-10-09T05:51:37.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:37 smithi125 bash[27513]: audit 2023-10-09T05:51:36.771825+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi125", "addr": "172.21.15.125", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:38.817 INFO:teuthology.orchestra.run.smithi125.stderr:Deploying unmanaged mon service... 2023-10-09T05:51:38.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:38 smithi125 bash[27513]: audit 2023-10-09T05:51:37.598875+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:38.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:38 smithi125 bash[27513]: cephadm 2023-10-09T05:51:37.599449+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi125 2023-10-09T05:51:38.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:38 smithi125 bash[27513]: audit 2023-10-09T05:51:37.605752+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:40.654 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-09T05:51:41.387 INFO:teuthology.orchestra.run.smithi125.stderr:Deploying unmanaged mgr service... 2023-10-09T05:51:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:41 smithi125 bash[27513]: audit 2023-10-09T05:51:40.646240+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:41 smithi125 bash[27513]: cephadm 2023-10-09T05:51:40.647696+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-09T05:51:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:41 smithi125 bash[27513]: audit 2023-10-09T05:51:40.652606+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:41 smithi125 bash[27513]: audit 2023-10-09T05:51:40.959431+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:41 smithi125 bash[27513]: audit 2023-10-09T05:51:41.197584+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:42.660 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-09T05:51:44.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:43 smithi125 bash[27513]: audit 2023-10-09T05:51:42.651686+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:44.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:43 smithi125 bash[27513]: cephadm 2023-10-09T05:51:42.654052+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-09T05:51:44.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:43 smithi125 bash[27513]: audit 2023-10-09T05:51:42.657990+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:45.878 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:45 smithi125 bash[27513]: audit 2023-10-09T05:51:44.600032+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.125:0/752249137' entity='client.admin' 2023-10-09T05:51:47.199 INFO:teuthology.orchestra.run.smithi125.stderr:Enabling the dashboard module... 2023-10-09T05:51:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:47 smithi125 bash[27513]: cluster 2023-10-09T05:51:45.825470+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:51:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:47 smithi125 bash[27513]: audit 2023-10-09T05:51:46.191268+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.125:0/818987545' entity='mgr.y' 2023-10-09T05:51:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:47 smithi125 bash[27513]: audit 2023-10-09T05:51:46.500844+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.125:0/1225740200' entity='client.admin' 2023-10-09T05:51:49.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27513]: cluster 2023-10-09T05:51:47.825952+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:51:49.460 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27513]: audit 2023-10-09T05:51:48.660123+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.125:0/1499878355' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T05:51:49.460 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27779]: ignoring --setuser ceph since I am not root 2023-10-09T05:51:49.460 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27779]: ignoring --setgroup ceph since I am not root 2023-10-09T05:51:49.461 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27779]: debug 2023-10-09T05:51:49.396+0000 7f24616d9680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:51:49.884 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27779]: debug 2023-10-09T05:51:49.504+0000 7f24616d9680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:51:50.199 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:49 smithi125 bash[27779]: debug 2023-10-09T05:51:49.988+0000 7f24616d9680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:51:50.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:50 smithi125 bash[27513]: audit 2023-10-09T05:51:49.196667+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.125:0/1499878355' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T05:51:50.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:50 smithi125 bash[27513]: cluster 2023-10-09T05:51:49.196843+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 23s) 2023-10-09T05:51:51.277 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:50 smithi125 bash[27779]: debug 2023-10-09T05:51:50.924+0000 7f24616d9680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:51:51.277 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:51 smithi125 bash[27779]: debug 2023-10-09T05:51:51.024+0000 7f24616d9680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:51:51.277 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:51 smithi125 bash[27779]: debug 2023-10-09T05:51:51.272+0000 7f24616d9680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:51:51.977 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:51.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-09T05:51:51.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "available": true, 2023-10-09T05:51:51.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-09T05:51:51.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-09T05:51:51.978 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:52.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:52 smithi125 bash[27513]: audit 2023-10-09T05:51:51.976112+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.125:0/4294863696' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:51:52.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:52 smithi125 bash[27779]: debug 2023-10-09T05:51:52.440+0000 7f24616d9680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:51:52.728 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for the mgr to restart... 2023-10-09T05:51:52.728 INFO:teuthology.orchestra.run.smithi125.stderr:Waiting for mgr epoch 9... 2023-10-09T05:51:52.884 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:52 smithi125 bash[27779]: debug 2023-10-09T05:51:52.604+0000 7f24616d9680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:52.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:52 smithi125 bash[27779]: debug 2023-10-09T05:51:52.704+0000 7f24616d9680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:52.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:52 smithi125 bash[27779]: debug 2023-10-09T05:51:52.820+0000 7f24616d9680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:51:53.384 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:53 smithi125 bash[27779]: debug 2023-10-09T05:51:53.044+0000 7f24616d9680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:51:53.884 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:53 smithi125 bash[27779]: debug 2023-10-09T05:51:53.424+0000 7f24616d9680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:51:53.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:53 smithi125 bash[27779]: debug 2023-10-09T05:51:53.548+0000 7f24616d9680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:54.530 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:54 smithi125 bash[27779]: debug 2023-10-09T05:51:54.420+0000 7f24616d9680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:51:54.867 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:54 smithi125 bash[27779]: debug 2023-10-09T05:51:54.524+0000 7f24616d9680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:51:54.867 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:54 smithi125 bash[27779]: debug 2023-10-09T05:51:54.640+0000 7f24616d9680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:51:55.134 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:54 smithi125 bash[27779]: debug 2023-10-09T05:51:54.864+0000 7f24616d9680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:51:55.491 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:55 smithi125 bash[27779]: debug 2023-10-09T05:51:55.148+0000 7f24616d9680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:51:55.492 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:55 smithi125 bash[27779]: debug 2023-10-09T05:51:55.316+0000 7f24616d9680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:51:55.808 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:55 smithi125 bash[27779]: debug 2023-10-09T05:51:55.488+0000 7f24616d9680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:55.808 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:55 smithi125 bash[27779]: debug 2023-10-09T05:51:55.696+0000 7f24616d9680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:51:56.087 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:55 smithi125 bash[27779]: debug 2023-10-09T05:51:55.804+0000 7f24616d9680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:51:56.087 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27779]: Warning: Permanently added '172.21.15.125' (ECDSA) to the list of known hosts. 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: cluster 2023-10-09T05:51:55.829118+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: cluster 2023-10-09T05:51:55.829550+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: cluster 2023-10-09T05:51:55.836078+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: cluster 2023-10-09T05:51:55.836319+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00694098s) 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.837561+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.838980+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:51:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.840057+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.840326+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.840552+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: cluster 2023-10-09T05:51:55.844018+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.983605+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:55.985790+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:56.019710+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:51:56.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:56 smithi125 bash[27513]: audit 2023-10-09T05:51:56.023730+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:51:56.857 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: { 2023-10-09T05:51:56.857 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-09T05:51:56.857 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: "initialized": true 2023-10-09T05:51:56.858 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: } 2023-10-09T05:51:57.591 INFO:teuthology.orchestra.run.smithi125.stderr:mgr epoch 9 is available 2023-10-09T05:51:57.591 INFO:teuthology.orchestra.run.smithi125.stderr:Generating a dashboard self-signed certificate... 2023-10-09T05:51:57.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:57 smithi125 bash[27513]: audit 2023-10-09T05:51:56.838740+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:51:57.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:57 smithi125 bash[27513]: cluster 2023-10-09T05:51:56.840312+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.01093s) 2023-10-09T05:51:57.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:57 smithi125 bash[27513]: audit 2023-10-09T05:51:56.855382+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:51:58.936 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-09T05:51:59.622 INFO:teuthology.orchestra.run.smithi125.stderr:Creating initial admin user... 2023-10-09T05:52:00.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:59 smithi125 bash[27513]: cluster 2023-10-09T05:51:58.840560+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 3s) 2023-10-09T05:52:00.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:59 smithi125 bash[27513]: audit 2023-10-09T05:51:58.899167+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:00.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:59 smithi125 bash[27513]: audit 2023-10-09T05:51:58.930033+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:00.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:51:59 smithi125 bash[27513]: audit 2023-10-09T05:51:58.934579+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:01.202 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$1OJRm9RnSe906mjiTPg3DuXkJm/FVx4ZIYvVgVv0mkVVPqUASeBea", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696830721, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T05:52:01.885 INFO:teuthology.orchestra.run.smithi125.stderr:Fetching dashboard port number... 2023-10-09T05:52:02.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:02 smithi125 bash[27513]: audit 2023-10-09T05:52:00.948928+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:02.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:02 smithi125 bash[27513]: audit 2023-10-09T05:52:01.200299+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:03.270 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: 8443 2023-10-09T05:52:03.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:03 smithi125 bash[27513]: cluster 2023-10-09T05:52:02.204318+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-09T05:52:03.532 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:03 smithi125 bash[27513]: audit 2023-10-09T05:52:02.918865+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:03.982 INFO:teuthology.orchestra.run.smithi125.stderr:firewalld does not appear to be present 2023-10-09T05:52:03.982 INFO:teuthology.orchestra.run.smithi125.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr:Ceph Dashboard is now available at: 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr: URL: https://smithi125.front.sepia.ceph.com:8443/ 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr: User: admin 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr: Password: 5acj04ntng 2023-10-09T05:52:03.985 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:03.986 INFO:teuthology.orchestra.run.smithi125.stderr:Enabling autotune for osd_memory_target 2023-10-09T05:52:04.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:04 smithi125 bash[27513]: audit 2023-10-09T05:52:03.268807+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.125:0/2196427102' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T05:52:07.463 INFO:teuthology.orchestra.run.smithi125.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-09T05:52:08.481 INFO:teuthology.orchestra.run.smithi125.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T05:52:08.481 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.481 INFO:teuthology.orchestra.run.smithi125.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr:Or, if you are only running a single cluster on this host: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: ceph telemetry on 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.482 INFO:teuthology.orchestra.run.smithi125.stderr:For more information see: 2023-10-09T05:52:08.483 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.483 INFO:teuthology.orchestra.run.smithi125.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-09T05:52:08.483 INFO:teuthology.orchestra.run.smithi125.stderr: 2023-10-09T05:52:08.483 INFO:teuthology.orchestra.run.smithi125.stderr:Bootstrap complete. 2023-10-09T05:52:08.522 INFO:tasks.cephadm:Fetching config... 2023-10-09T05:52:08.523 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:08.523 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T05:52:08.529 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T05:52:08.529 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:08.529 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T05:52:08.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:08 smithi125 bash[27513]: audit 2023-10-09T05:52:07.457046+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.125:0/2445918969' entity='client.admin' 2023-10-09T05:52:08.572 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T05:52:08.572 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:08.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T05:52:08.632 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T05:52:08.632 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:08.632 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T05:52:08.680 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T05:52:08.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpJziRse2FgxMF5YKXEOkNOqKFmqy5fjTAMiz5WeRtbTjXkC/MKpfXAqvFT31VHqwGBIRZfgjvs+EzDxm+Fro+4G25JizzW362gBhXyd6Tlvqf2+Q9b7ikZtSTYrNOcVRAcfcmJBioRZlk0XLmv6LF/mk8aFB9VHnW6itY9HbCXDpQQaRXT/viNG2v/GzyXC+YIhZVxGCasxdRk629kL09ZBCDuWMMBvpZXGGqf7GG3MAVGhPk713QsJgnbtbQrNvEClVtn2FpFTXxej2tlr0xAinOB46mQuRaY1dnZB8sBIpL2kOBx+U+Dz6i8bHlr4CnPrjWTaM5OQ4bBQyXP11ZMroiLMwjryZuHsYrEbv7xo0AxuxV5haxocsh58OzUJr+PPobECmX65DkttA8W2MoSqUAQ4rEm0iFhYcvPon20j+D8MZonjuDspMxz0nI36sjyuLMZUS9VFL+/2Rr5Y5tA5FoHkTMvGPX0edFkPomkSlY5PoaDcjRFpePjrHr07U= ceph-95d0291c-6667-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:52:08.745 INFO:teuthology.orchestra.run.smithi125.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpJziRse2FgxMF5YKXEOkNOqKFmqy5fjTAMiz5WeRtbTjXkC/MKpfXAqvFT31VHqwGBIRZfgjvs+EzDxm+Fro+4G25JizzW362gBhXyd6Tlvqf2+Q9b7ikZtSTYrNOcVRAcfcmJBioRZlk0XLmv6LF/mk8aFB9VHnW6itY9HbCXDpQQaRXT/viNG2v/GzyXC+YIhZVxGCasxdRk629kL09ZBCDuWMMBvpZXGGqf7GG3MAVGhPk713QsJgnbtbQrNvEClVtn2FpFTXxej2tlr0xAinOB46mQuRaY1dnZB8sBIpL2kOBx+U+Dz6i8bHlr4CnPrjWTaM5OQ4bBQyXP11ZMroiLMwjryZuHsYrEbv7xo0AxuxV5haxocsh58OzUJr+PPobECmX65DkttA8W2MoSqUAQ4rEm0iFhYcvPon20j+D8MZonjuDspMxz0nI36sjyuLMZUS9VFL+/2Rr5Y5tA5FoHkTMvGPX0edFkPomkSlY5PoaDcjRFpePjrHr07U= ceph-95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:08.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpJziRse2FgxMF5YKXEOkNOqKFmqy5fjTAMiz5WeRtbTjXkC/MKpfXAqvFT31VHqwGBIRZfgjvs+EzDxm+Fro+4G25JizzW362gBhXyd6Tlvqf2+Q9b7ikZtSTYrNOcVRAcfcmJBioRZlk0XLmv6LF/mk8aFB9VHnW6itY9HbCXDpQQaRXT/viNG2v/GzyXC+YIhZVxGCasxdRk629kL09ZBCDuWMMBvpZXGGqf7GG3MAVGhPk713QsJgnbtbQrNvEClVtn2FpFTXxej2tlr0xAinOB46mQuRaY1dnZB8sBIpL2kOBx+U+Dz6i8bHlr4CnPrjWTaM5OQ4bBQyXP11ZMroiLMwjryZuHsYrEbv7xo0AxuxV5haxocsh58OzUJr+PPobECmX65DkttA8W2MoSqUAQ4rEm0iFhYcvPon20j+D8MZonjuDspMxz0nI36sjyuLMZUS9VFL+/2Rr5Y5tA5FoHkTMvGPX0edFkPomkSlY5PoaDcjRFpePjrHr07U= ceph-95d0291c-6667-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:52:08.777 INFO:teuthology.orchestra.run.smithi155.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpJziRse2FgxMF5YKXEOkNOqKFmqy5fjTAMiz5WeRtbTjXkC/MKpfXAqvFT31VHqwGBIRZfgjvs+EzDxm+Fro+4G25JizzW362gBhXyd6Tlvqf2+Q9b7ikZtSTYrNOcVRAcfcmJBioRZlk0XLmv6LF/mk8aFB9VHnW6itY9HbCXDpQQaRXT/viNG2v/GzyXC+YIhZVxGCasxdRk629kL09ZBCDuWMMBvpZXGGqf7GG3MAVGhPk713QsJgnbtbQrNvEClVtn2FpFTXxej2tlr0xAinOB46mQuRaY1dnZB8sBIpL2kOBx+U+Dz6i8bHlr4CnPrjWTaM5OQ4bBQyXP11ZMroiLMwjryZuHsYrEbv7xo0AxuxV5haxocsh58OzUJr+PPobECmX65DkttA8W2MoSqUAQ4rEm0iFhYcvPon20j+D8MZonjuDspMxz0nI36sjyuLMZUS9VFL+/2Rr5Y5tA5FoHkTMvGPX0edFkPomkSlY5PoaDcjRFpePjrHr07U= ceph-95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:08.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T05:52:13.556 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T05:52:13.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T05:52:13.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:13 smithi125 bash[27513]: audit 2023-10-09T05:52:12.526334+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.125:0/2140589982' entity='client.admin' 2023-10-09T05:52:16.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: audit 2023-10-09T05:52:15.515955+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: audit 2023-10-09T05:52:15.839646+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi125", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:52:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: cluster 2023-10-09T05:52:15.841529+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: audit 2023-10-09T05:52:15.847583+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: audit 2023-10-09T05:52:15.850716+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:16 smithi125 bash[27513]: audit 2023-10-09T05:52:15.859540+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:17.809 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi155 2023-10-09T05:52:17.809 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:52:17.809 DEBUG:teuthology.orchestra.run.smithi155:> dd of=/etc/ceph/ceph.conf 2023-10-09T05:52:17.817 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:52:17.817 DEBUG:teuthology.orchestra.run.smithi155:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:17.867 INFO:tasks.cephadm:Adding host smithi155 to orchestrator... 2023-10-09T05:52:17.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi155 2023-10-09T05:52:18.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.066817+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:18.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.077908+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.080498+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.081933+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: cephadm 2023-10-09T05:52:17.083761+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: cephadm 2023-10-09T05:52:17.105991+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.135585+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:18.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:18 smithi125 bash[27513]: audit 2023-10-09T05:52:17.140106+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:19.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:19 smithi125 bash[27513]: cluster 2023-10-09T05:52:17.842028+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:21.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:21 smithi125 bash[27513]: cluster 2023-10-09T05:52:19.842550+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:21.385 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:52:21 smithi125 bash[27779]: Warning: Permanently added 'smithi155,172.21.15.155' (ECDSA) to the list of known hosts. 2023-10-09T05:52:21.841 INFO:teuthology.orchestra.run.smithi125.stdout:Added host 'smithi155' with addr '172.21.15.155' 2023-10-09T05:52:22.202 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:52:21 smithi125 bash[27779]: Warning: Permanently added '172.21.15.155' (ECDSA) to the list of known hosts. 2023-10-09T05:52:22.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:22 smithi125 bash[27513]: audit 2023-10-09T05:52:20.983750+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi155", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:22.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:22 smithi125 bash[27513]: audit 2023-10-09T05:52:21.839488+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:22.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:22 smithi125 bash[27513]: audit 2023-10-09T05:52:21.868833+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:22.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T05:52:23.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:23 smithi125 bash[27513]: cephadm 2023-10-09T05:52:21.840374+0000 mgr.y (mgr.14146) 12 : cephadm [INF] Added host smithi155 2023-10-09T05:52:23.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:23 smithi125 bash[27513]: cluster 2023-10-09T05:52:21.843049+0000 mgr.y (mgr.14146) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:23.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:23 smithi125 bash[27513]: audit 2023-10-09T05:52:22.421134+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:24.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:24 smithi125 bash[27513]: audit 2023-10-09T05:52:23.695496+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:24.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:24 smithi125 bash[27513]: cluster 2023-10-09T05:52:23.843520+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:25.708 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T05:52:25.708 INFO:teuthology.orchestra.run.smithi125.stdout:[{"addr": "172.21.15.125", "hostname": "smithi125", "labels": [], "status": ""}, {"addr": "172.21.15.155", "hostname": "smithi155", "labels": [], "status": ""}] 2023-10-09T05:52:26.403 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T05:52:26.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T05:52:27.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:27 smithi125 bash[27513]: audit 2023-10-09T05:52:25.705845+0000 mgr.y (mgr.14146) 15 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:27.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:27 smithi125 bash[27513]: cluster 2023-10-09T05:52:25.843954+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:29.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:29 smithi125 bash[27513]: cluster 2023-10-09T05:52:27.844423+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:29 smithi125 bash[27513]: audit 2023-10-09T05:52:28.983590+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:30.338 INFO:teuthology.orchestra.run.smithi125.stderr:adjusted tunables profile to default 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.266618+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.267722+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi155", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.268958+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.270033+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: cephadm 2023-10-09T05:52:29.271757+0000 mgr.y (mgr.14146) 18 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:52:30.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: cephadm 2023-10-09T05:52:29.287312+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:30.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.317425+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:30.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.321897+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:30.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: audit 2023-10-09T05:52:29.391384+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.125:0/3186023801' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:52:30.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:30 smithi125 bash[27513]: cluster 2023-10-09T05:52:29.844826+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:31.078 INFO:tasks.cephadm:Adding mon.a on smithi125 2023-10-09T05:52:31.078 INFO:tasks.cephadm:Adding mon.c on smithi125 2023-10-09T05:52:31.079 INFO:tasks.cephadm:Adding mon.b on smithi155 2023-10-09T05:52:31.079 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b' 2023-10-09T05:52:31.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:31 smithi125 bash[27513]: audit 2023-10-09T05:52:30.333747+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.125:0/3186023801' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:52:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:31 smithi125 bash[27513]: cluster 2023-10-09T05:52:30.333800+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:32.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:32 smithi125 bash[27513]: cluster 2023-10-09T05:52:31.845191+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:33.876 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled mon update... 2023-10-09T05:52:34.843 DEBUG:teuthology.orchestra.run.smithi125:mon.c> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T05:52:34.846 DEBUG:teuthology.orchestra.run.smithi155:mon.b> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T05:52:34.850 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:52:34.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:52:35.015 INFO:journalctl@ceph.mon.c.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:52:35.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: cluster 2023-10-09T05:52:33.845650+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.865123+0000 mgr.y (mgr.14146) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: cephadm 2023-10-09T05:52:33.871214+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Saving service mon spec with placement smithi125:172.21.15.125=a;smithi125:[v2:172.21.15.125:3301,v1:172.21.15.125:6790]=c;smithi155:172.21.15.155=b;count:3 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.874765+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.900799+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.904336+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.905872+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.910898+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.913544+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:35.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: audit 2023-10-09T05:52:33.914836+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:35.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:34 smithi125 bash[27513]: cephadm 2023-10-09T05:52:33.915947+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Deploying daemon mon.b on smithi155 2023-10-09T05:52:35.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:52:37.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:37 smithi125 bash[27513]: cluster 2023-10-09T05:52:35.846084+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T05:52:39.433 INFO:teuthology.orchestra.run.smithi155.stdout:{"epoch":1,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:50:54.893889Z","created":"2023-10-09T05:50:54.893889Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3300","nonce":0},{"type":"v1","addr":"172.21.15.125:6789","nonce":0}]},"addr":"172.21.15.125:6789/0","public_addr":"172.21.15.125:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T05:52:39.435 INFO:teuthology.orchestra.run.smithi155.stderr:dumped monmap epoch 1 2023-10-09T05:52:39.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:39 smithi125 bash[27513]: cluster 2023-10-09T05:52:37.846530+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:39.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:39 smithi155 systemd[1]: Started Ceph mon.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:52:40.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:40 smithi125 bash[27513]: audit 2023-10-09T05:52:39.431597+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.155:0/3035401237' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:40.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:40 smithi125 bash[27513]: audit 2023-10-09T05:52:39.806609+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:40.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:40 smithi125 bash[27513]: audit 2023-10-09T05:52:39.808439+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:40.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:40 smithi125 bash[27513]: audit 2023-10-09T05:52:39.809977+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.802+0000 7f38afff2880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.802+0000 7f38afff2880 0 ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable), process ceph-mon, pid 7 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.802+0000 7f38afff2880 0 pidfile_write: ignore empty --pid-file 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.806+0000 7f38afff2880 0 load: jerasure load: lrc load: isa 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: RocksDB version: 6.8.1 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Compile date Oct 5 2023 2023-10-09T05:52:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: DB SUMMARY 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: CURRENT file: CURRENT 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.error_if_exists: 0 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.create_if_missing: 0 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.env: 0x55fa19fc1080 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.fs: Posix File System 2023-10-09T05:52:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.info_log: 0x55fa1bc67a80 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.statistics: (nil) 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.use_fsync: 0 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:52:41.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.db_log_dir: 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:52:41.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.write_buffer_manager: 0x55fa1c9642d0 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:52:41.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.unordered_write: 0 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:52:41.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.row_cache: None 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.wal_filter: None 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.preserve_deletes: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.two_write_queues: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.atomic_flush: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:52:41.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:52:41.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_open_files: -1 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Compression algorithms supported: 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kZSTD supported: 0 2023-10-09T05:52:41.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kXpressCompression supported: 0 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kZlibCompression supported: 1 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.merge_operator: 2023-10-09T05:52:41.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_filter: None 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fa1bc15d18) 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: cache_index_and_filter_blocks: 1 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: pin_top_level_index_and_filter: 1 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: index_type: 0 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: data_block_index_type: 0 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: index_shortening: 1 2023-10-09T05:52:41.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: hash_index_allow_collision: 1 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: checksum: 1 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: no_block_cache: 0 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_cache: 0x55fa1bc4cf10 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_cache_name: BinnedLRUCache 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_cache_options: 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: capacity : 536870912 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: num_shard_bits : 4 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: strict_capacity_limit : 0 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: high_pri_pool_ratio: 0.000 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_cache_compressed: (nil) 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: persistent_cache: (nil) 2023-10-09T05:52:41.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_size: 4096 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_size_deviation: 10 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_restart_interval: 16 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: index_block_restart_interval: 1 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: metadata_block_size: 4096 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: partition_filters: 0 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: use_delta_encoding: 1 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: whole_key_filtering: 1 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: verify_compression: 0 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: read_amp_bytes_per_bit: 0 2023-10-09T05:52:41.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: format_version: 2 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: enable_index_compression: 1 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: block_align: 0 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression: NoCompression 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.num_levels: 7 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:52:41.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:52:41.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:52:41.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:52:41.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:52:41.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.table_properties_collectors: 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.bloom_locality: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:52:41.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.ttl: 2592000 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830760816207, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830760816918, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696830760, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-09T05:52:41.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.810+0000 7f38afff2880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696830760817534, "job": 1, "event": "recovery_finished"} 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f38afff2880 4 rocksdb: DB pointer 0x55fa1bd15800 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f3896df9700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f3896df9700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** DB Stats ** 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:41.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** Compaction Stats [default] ** 2023-10-09T05:52:41.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:41.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:41.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** Compaction Stats [default] ** 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:41.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Cumulative compaction: 0.00 GB write, 0.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Interval compaction: 0.00 GB write, 0.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** Compaction Stats [default] ** 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** Compaction Stats [default] ** 2023-10-09T05:52:41.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.1 0.00 0.00 1 0.001 0 0 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Cumulative compaction: 0.00 GB write, 0.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:41.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f38afff2880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f38afff2880 0 using public_addr v2:172.21.15.155:0/0 -> [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.814+0000 7f38afff2880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] at bind addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.818+0000 7f38afff2880 1 mon.b@-1(???) e0 preinit fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: e1 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: legacy client fscid: -1 2023-10-09T05:52:41.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: No filesystems configured 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:41.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: audit 2023-10-09T05:52:39.431597+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.155:0/3035401237' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: audit 2023-10-09T05:52:39.806609+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: audit 2023-10-09T05:52:39.808439+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: audit 2023-10-09T05:52:39.809977+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:41.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:40 smithi155 bash[27817]: debug 2023-10-09T05:52:40.842+0000 7f38995fe700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:52:41.479 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:52:41.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:52:45.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:44 smithi125 bash[33919]: debug 2023-10-09T05:52:44.795+0000 7f97d020c700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:40.858255+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:40.858569+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:40.879692+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:41.847117+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:41.853920+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:42.854362+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:42.855864+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:43.847410+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:43.853817+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:44.806308+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:44.854770+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:45.806685+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:45.854563+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.900023+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.905546+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0]} removed_ranks: {} 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.923345+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.923404+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.923764+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 50s) 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: cluster 2023-10-09T05:52:45.930707+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:45.935479+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:46.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:45 smithi125 bash[27513]: audit 2023-10-09T05:52:45.937568+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:46.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:40.858255+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:40.858569+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:40.879692+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:41.847117+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:41.853920+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:42.854362+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:42.855864+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:43.847410+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:43.853817+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:44.806308+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:44.854770+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:45.806685+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:45.854563+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.900023+0000 mon.a (mon.0) 137 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.905546+0000 mon.a (mon.0) 138 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0]} removed_ranks: {} 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.923345+0000 mon.a (mon.0) 139 : cluster [DBG] fsmap 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.923404+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.923764+0000 mon.a (mon.0) 141 : cluster [DBG] mgrmap e13: y(active, since 50s) 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: cluster 2023-10-09T05:52:45.930707+0000 mon.a (mon.0) 142 : cluster [INF] overall HEALTH_OK 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:45.935479+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:46.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:45 smithi155 bash[27817]: audit 2023-10-09T05:52:45.937568+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:47.134 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:52:46 smithi125 bash[27779]: debug 2023-10-09T05:52:46.851+0000 7f2414ede700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T05:52:52.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:46.812377+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:52.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:46.812706+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:52.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:46.812787+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:52.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:46.814103+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:46.814301+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:47.807053+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:47.848254+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:48.807266+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:49.806764+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:49.848709+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:50.807375+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.807411+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.814425+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.819748+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.839392+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.839444+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.839752+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 56s) 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.840021+0000 mon.a (mon.0) 161 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.846229+0000 mon.a (mon.0) 162 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.846268+0000 mon.a (mon.0) 163 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.846291+0000 mon.a (mon.0) 164 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] is down (out of quorum) 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.852281+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.856980+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.858199+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.859860+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:52.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:51 smithi125 bash[27513]: audit 2023-10-09T05:52:51.886867+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:52.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:46.812377+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:52.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:46.812706+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:46.812787+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:46.814103+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:46.814301+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:47.807053+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:47.848254+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:48.807266+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:49.806764+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:49.848709+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:50.807375+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.807411+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.814425+0000 mon.a (mon.0) 156 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.819748+0000 mon.a (mon.0) 157 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.839392+0000 mon.a (mon.0) 158 : cluster [DBG] fsmap 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.839444+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.839752+0000 mon.a (mon.0) 160 : cluster [DBG] mgrmap e13: y(active, since 56s) 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.840021+0000 mon.a (mon.0) 161 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.846229+0000 mon.a (mon.0) 162 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.846268+0000 mon.a (mon.0) 163 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.846291+0000 mon.a (mon.0) 164 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] is down (out of quorum) 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.852281+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.856980+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.858199+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.859860+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:52.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:51 smithi155 bash[27817]: audit 2023-10-09T05:52:51.886867+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:53.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: cluster 2023-10-09T05:52:51.849085+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:53.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: cephadm 2023-10-09T05:52:51.861822+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:52:53.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: cephadm 2023-10-09T05:52:51.862371+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:52:53.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:51.891747+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:51.904944+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: cephadm 2023-10-09T05:52:51.907385+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:51.907867+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:51.909125+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:51.910281+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: cephadm 2023-10-09T05:52:51.911322+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi125 2023-10-09T05:52:53.238 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:52 smithi125 bash[27513]: audit 2023-10-09T05:52:52.807573+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:53.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: cluster 2023-10-09T05:52:51.849085+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:53.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: cephadm 2023-10-09T05:52:51.861822+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: cephadm 2023-10-09T05:52:51.862371+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:51.891747+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:51.904944+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: cephadm 2023-10-09T05:52:51.907385+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:51.907867+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:51.909125+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:51.910281+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: cephadm 2023-10-09T05:52:51.911322+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi125 2023-10-09T05:52:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:52 smithi155 bash[27817]: audit 2023-10-09T05:52:52.807573+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:53.606 INFO:teuthology.orchestra.run.smithi155.stdout: 2023-10-09T05:52:53.607 INFO:teuthology.orchestra.run.smithi155.stdout:{"epoch":3,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:52:46.807238Z","created":"2023-10-09T05:50:54.893889Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3300","nonce":0},{"type":"v1","addr":"172.21.15.125:6789","nonce":0}]},"addr":"172.21.15.125:6789/0","public_addr":"172.21.15.125:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:3300","nonce":0},{"type":"v1","addr":"172.21.15.155:6789","nonce":0}]},"addr":"172.21.15.155:6789/0","public_addr":"172.21.15.155:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:3301","nonce":0},{"type":"v1","addr":"172.21.15.125:6790","nonce":0}]},"addr":"172.21.15.125:6790/0","public_addr":"172.21.15.125:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T05:52:53.612 INFO:teuthology.orchestra.run.smithi155.stderr:dumped monmap epoch 3 2023-10-09T05:52:54.126 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:48.808860+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.840834+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.841290+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.841776+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.865640+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.874004+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.874609+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.874649+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.874887+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.875066+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:52:54.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.875091+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:51.849085+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cephadm 2023-10-09T05:52:51.861822+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cephadm 2023-10-09T05:52:51.862371+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:51.891747+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:51.904944+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cephadm 2023-10-09T05:52:51.907385+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:51.907867+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:51.909125+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:51.910281+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:54.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cephadm 2023-10-09T05:52:51.911322+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi125 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: audit 2023-10-09T05:52:52.807573+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:48.808860+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.840834+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.841290+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.841776+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.865640+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.874004+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.874609+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.874649+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.874887+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.875066+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:52:54.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:53 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.875091+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:48.808860+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.840834+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.841290+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.841776+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.865640+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.874004+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T05:52:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.874609+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-09T05:52:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.874649+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.874887+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 58s) 2023-10-09T05:52:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.875066+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T05:52:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:53 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.875091+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-09T05:52:54.401 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T05:52:54.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T05:52:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.849547+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:55.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: cluster 2023-10-09T05:52:53.901142+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: audit 2023-10-09T05:52:53.913789+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: cephadm 2023-10-09T05:52:53.914560+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: audit 2023-10-09T05:52:53.914876+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: audit 2023-10-09T05:52:53.915776+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: audit 2023-10-09T05:52:53.916598+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: cephadm 2023-10-09T05:52:53.917310+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi125 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[27513]: audit 2023-10-09T05:52:54.807444+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:55.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.849547+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: cluster 2023-10-09T05:52:53.901142+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: audit 2023-10-09T05:52:53.913789+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: cephadm 2023-10-09T05:52:53.914560+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: audit 2023-10-09T05:52:53.914876+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: audit 2023-10-09T05:52:53.915776+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: audit 2023-10-09T05:52:53.916598+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: cephadm 2023-10-09T05:52:53.917310+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi125 2023-10-09T05:52:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:54 smithi125 bash[33919]: audit 2023-10-09T05:52:54.807444+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.849547+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: cluster 2023-10-09T05:52:53.901142+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-09T05:52:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: audit 2023-10-09T05:52:53.913789+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: cephadm 2023-10-09T05:52:53.914560+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-09T05:52:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: audit 2023-10-09T05:52:53.914876+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: audit 2023-10-09T05:52:53.915776+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: audit 2023-10-09T05:52:53.916598+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: cephadm 2023-10-09T05:52:53.917310+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring daemon mon.c on smithi125 2023-10-09T05:52:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:54 smithi155 bash[27817]: audit 2023-10-09T05:52:54.807444+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:57.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[27513]: cluster 2023-10-09T05:52:55.849984+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[27513]: audit 2023-10-09T05:52:56.950075+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[27513]: audit 2023-10-09T05:52:56.951359+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[27513]: audit 2023-10-09T05:52:56.952540+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[27513]: audit 2023-10-09T05:52:56.953689+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[33919]: cluster 2023-10-09T05:52:55.849984+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[33919]: audit 2023-10-09T05:52:56.950075+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[33919]: audit 2023-10-09T05:52:56.951359+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:57.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[33919]: audit 2023-10-09T05:52:56.952540+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:57.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:57 smithi125 bash[33919]: audit 2023-10-09T05:52:56.953689+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:57 smithi155 bash[27817]: cluster 2023-10-09T05:52:55.849984+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:57 smithi155 bash[27817]: audit 2023-10-09T05:52:56.950075+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:57 smithi155 bash[27817]: audit 2023-10-09T05:52:56.951359+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:57 smithi155 bash[27817]: audit 2023-10-09T05:52:56.952540+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:57 smithi155 bash[27817]: audit 2023-10-09T05:52:56.953689+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.302 INFO:teuthology.orchestra.run.smithi125.stdout:# minimal ceph.conf for 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:58.303 INFO:teuthology.orchestra.run.smithi125.stdout:[global] 2023-10-09T05:52:58.303 INFO:teuthology.orchestra.run.smithi125.stdout: fsid = 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:58.303 INFO:teuthology.orchestra.run.smithi125.stdout: mon_host = [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] 2023-10-09T05:52:58.390 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:58 smithi155 bash[27817]: cephadm 2023-10-09T05:52:56.950965+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:52:58.390 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:58 smithi155 bash[27817]: cephadm 2023-10-09T05:52:56.954703+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi155 2023-10-09T05:52:58.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:58 smithi125 bash[27513]: cephadm 2023-10-09T05:52:56.950965+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:52:58.540 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:58 smithi125 bash[27513]: cephadm 2023-10-09T05:52:56.954703+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi155 2023-10-09T05:52:58.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:58 smithi125 bash[33919]: cephadm 2023-10-09T05:52:56.950965+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-09T05:52:58.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:58 smithi125 bash[33919]: cephadm 2023-10-09T05:52:56.954703+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring daemon mon.b on smithi155 2023-10-09T05:52:58.997 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T05:52:58.997 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:58.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:52:59.048 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:52:59.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:59.108 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:52:59.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:52:59.123 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:52:59.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:59.180 INFO:tasks.cephadm:Adding mgr.y on smithi125 2023-10-09T05:52:59.180 INFO:tasks.cephadm:Adding mgr.x on smithi155 2023-10-09T05:52:59.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi125=y;smithi155=x' 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: cluster 2023-10-09T05:52:57.850477+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.301647+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.125:0/2226495088' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.822623+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.824311+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.826698+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.827713+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[27513]: audit 2023-10-09T05:52:58.832915+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:59.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: cluster 2023-10-09T05:52:57.850477+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.301647+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.125:0/2226495088' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.822623+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.824311+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.826698+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.827713+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:52:59 smithi125 bash[33919]: audit 2023-10-09T05:52:58.832915+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: cluster 2023-10-09T05:52:57.850477+0000 mgr.y (mgr.14146) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.301647+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.125:0/2226495088' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.822623+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:52:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.824311+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.826698+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.827713+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:52:59 smithi155 bash[27817]: audit 2023-10-09T05:52:58.832915+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:01.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:01 smithi155 bash[27817]: cluster 2023-10-09T05:52:59.850928+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:01.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:01 smithi125 bash[27513]: cluster 2023-10-09T05:52:59.850928+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:01.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:01 smithi125 bash[33919]: cluster 2023-10-09T05:52:59.850928+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:02.157 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled mgr update... 2023-10-09T05:53:03.163 DEBUG:teuthology.orchestra.run.smithi155:mgr.x> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T05:53:03.165 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T05:53:03.165 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:53:03.165 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:53:03.170 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:53:03.171 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_1 2023-10-09T05:53:03.221 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:53:03.221 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 5h/5d Inode: 489 Links: 1 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 05:45:50.217585469 +0000 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 05:45:49.889757036 +0000 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 05:45:49.889757036 +0000 2023-10-09T05:53:03.222 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T05:53:03.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:53:03.274 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T05:53:03.274 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T05:53:03.274 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000358662 s, 1.4 MB/s 2023-10-09T05:53:03.275 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:53:03.327 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_2 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 05:45:50.457459933 +0000 2023-10-09T05:53:03.378 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 05:45:50.453462025 +0000 2023-10-09T05:53:03.379 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 05:45:50.453462025 +0000 2023-10-09T05:53:03.379 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T05:53:03.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: cluster 2023-10-09T05:53:01.851353+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.146614+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: cephadm 2023-10-09T05:53:02.149847+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.155753+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.157734+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.160077+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.161172+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:03.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.167010+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.168955+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.172894+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.174000+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: audit 2023-10-09T05:53:02.175238+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[27513]: cephadm 2023-10-09T05:53:02.176220+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T05:53:03.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: cluster 2023-10-09T05:53:01.851353+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.146614+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: cephadm 2023-10-09T05:53:02.149847+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.155753+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.157734+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.160077+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.161172+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.167010+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.168955+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:03.435 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.172894+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:03.436 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.174000+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:03.436 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: audit 2023-10-09T05:53:02.175238+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.436 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:03 smithi125 bash[33919]: cephadm 2023-10-09T05:53:02.176220+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T05:53:03.437 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T05:53:03.437 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T05:53:03.437 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000412363 s, 1.2 MB/s 2023-10-09T05:53:03.437 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:53:03.487 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_3 2023-10-09T05:53:03.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: cluster 2023-10-09T05:53:01.851353+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:03.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.146614+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi125=y;smithi155=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:03.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: cephadm 2023-10-09T05:53:02.149847+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi125=y;smithi155=x;count:2 2023-10-09T05:53:03.491 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.155753+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.157734+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.160077+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.161172+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.167010+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.168955+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.172894+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.174000+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: audit 2023-10-09T05:53:02.175238+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:03.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:03 smithi155 bash[27817]: cephadm 2023-10-09T05:53:02.176220+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T05:53:03.537 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:53:03.537 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.537 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 5h/5d Inode: 501 Links: 1 2023-10-09T05:53:03.538 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.538 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 05:45:51.297054447 +0000 2023-10-09T05:53:03.538 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 05:45:50.989181661 +0000 2023-10-09T05:53:03.538 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 05:45:50.989181661 +0000 2023-10-09T05:53:03.538 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T05:53:03.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:53:03.573 INFO:journalctl@ceph.mgr.x.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:53:03.593 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T05:53:03.593 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T05:53:03.593 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.000405933 s, 1.3 MB/s 2023-10-09T05:53:03.594 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:53:03.647 DEBUG:teuthology.orchestra.run.smithi125:> stat /dev/vg_nvme/lv_4 2023-10-09T05:53:03.697 INFO:teuthology.orchestra.run.smithi125.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:53:03.697 INFO:teuthology.orchestra.run.smithi125.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.697 INFO:teuthology.orchestra.run.smithi125.stdout:Device: 5h/5d Inode: 507 Links: 1 2023-10-09T05:53:03.697 INFO:teuthology.orchestra.run.smithi125.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.697 INFO:teuthology.orchestra.run.smithi125.stdout:Access: 2023-10-09 05:45:51.800848781 +0000 2023-10-09T05:53:03.698 INFO:teuthology.orchestra.run.smithi125.stdout:Modify: 2023-10-09 05:45:51.496972833 +0000 2023-10-09T05:53:03.698 INFO:teuthology.orchestra.run.smithi125.stdout:Change: 2023-10-09 05:45:51.496972833 +0000 2023-10-09T05:53:03.698 INFO:teuthology.orchestra.run.smithi125.stdout: Birth: - 2023-10-09T05:53:03.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:53:03.753 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records in 2023-10-09T05:53:03.753 INFO:teuthology.orchestra.run.smithi125.stderr:1+0 records out 2023-10-09T05:53:03.753 INFO:teuthology.orchestra.run.smithi125.stderr:512 bytes copied, 0.00038186 s, 1.3 MB/s 2023-10-09T05:53:03.754 DEBUG:teuthology.orchestra.run.smithi125:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:53:03.811 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:53:03.811 DEBUG:teuthology.orchestra.run.smithi155:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:53:03.816 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:53:03.816 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_1 2023-10-09T05:53:03.863 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 5h/5d Inode: 493 Links: 1 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 05:45:52.162864908 +0000 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 05:45:51.846872193 +0000 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 05:45:51.846872193 +0000 2023-10-09T05:53:03.864 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T05:53:03.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:53:03.919 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T05:53:03.920 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T05:53:03.920 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000425833 s, 1.2 MB/s 2023-10-09T05:53:03.921 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:53:03.973 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_2 2023-10-09T05:53:04.023 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 5h/5d Inode: 499 Links: 1 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 05:45:52.682852922 +0000 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 05:45:52.378859930 +0000 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 05:45:52.378859930 +0000 2023-10-09T05:53:04.024 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T05:53:04.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:53:04.078 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T05:53:04.078 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T05:53:04.078 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000365942 s, 1.4 MB/s 2023-10-09T05:53:04.079 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:53:04.129 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_3 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 05:45:53.218840568 +0000 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 05:45:52.894848036 +0000 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 05:45:52.894848036 +0000 2023-10-09T05:53:04.180 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T05:53:04.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:53:04.235 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T05:53:04.235 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T05:53:04.235 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000388032 s, 1.3 MB/s 2023-10-09T05:53:04.236 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:53:04.289 DEBUG:teuthology.orchestra.run.smithi155:> stat /dev/vg_nvme/lv_4 2023-10-09T05:53:04.338 INFO:teuthology.orchestra.run.smithi155.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:53:04.338 INFO:teuthology.orchestra.run.smithi155.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout:Device: 5h/5d Inode: 511 Links: 1 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout:Access: 2023-10-09 05:45:53.742828490 +0000 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout:Modify: 2023-10-09 05:45:53.422835865 +0000 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout:Change: 2023-10-09 05:45:53.422835865 +0000 2023-10-09T05:53:04.339 INFO:teuthology.orchestra.run.smithi155.stdout: Birth: - 2023-10-09T05:53:04.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:53:04.389 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records in 2023-10-09T05:53:04.389 INFO:teuthology.orchestra.run.smithi155.stderr:1+0 records out 2023-10-09T05:53:04.389 INFO:teuthology.orchestra.run.smithi155.stderr:512 bytes copied, 0.000310356 s, 1.6 MB/s 2023-10-09T05:53:04.390 DEBUG:teuthology.orchestra.run.smithi155:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:53:04.442 INFO:tasks.cephadm:Deploying osd.0 on smithi125 with /dev/vg_nvme/lv_4... 2023-10-09T05:53:04.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:53:04.825 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:04 smithi155 systemd[1]: Started Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:53:05.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[27817]: cluster 2023-10-09T05:53:03.851785+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:05.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[27817]: audit 2023-10-09T05:53:04.800497+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:05.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[27817]: audit 2023-10-09T05:53:04.804684+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[27513]: cluster 2023-10-09T05:53:03.851785+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[27513]: audit 2023-10-09T05:53:04.800497+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[27513]: audit 2023-10-09T05:53:04.804684+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[33919]: cluster 2023-10-09T05:53:03.851785+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[33919]: audit 2023-10-09T05:53:04.800497+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:05.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:05 smithi125 bash[33919]: audit 2023-10-09T05:53:04.804684+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:05.866 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[28688]: debug 2023-10-09T05:53:05.705+0000 7fef55d1b700 1 -- 172.21.15.155:0/3028396255 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55be127b44e0 con 0x55be128a8c00 2023-10-09T05:53:06.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[28688]: debug 2023-10-09T05:53:05.865+0000 7fef66301680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:53:06.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:05 smithi155 bash[28688]: debug 2023-10-09T05:53:05.969+0000 7fef66301680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:53:06.892 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:06 smithi155 bash[28688]: debug 2023-10-09T05:53:06.425+0000 7fef66301680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:53:07.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:07 smithi155 bash[27817]: cluster 2023-10-09T05:53:05.852226+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.347 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:07 smithi155 bash[27817]: audit 2023-10-09T05:53:06.030384+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:07.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:07 smithi125 bash[33919]: cluster 2023-10-09T05:53:05.852226+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:07 smithi125 bash[33919]: audit 2023-10-09T05:53:06.030384+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:07.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:07 smithi125 bash[27513]: cluster 2023-10-09T05:53:05.852226+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:07 smithi125 bash[27513]: audit 2023-10-09T05:53:06.030384+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:07.642 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:07 smithi155 bash[28688]: debug 2023-10-09T05:53:07.345+0000 7fef66301680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:53:07.642 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:07 smithi155 bash[28688]: debug 2023-10-09T05:53:07.441+0000 7fef66301680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0166572 s, 630 MB/s 2023-10-09T05:53:07.887 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T05:53:08.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:07 smithi155 bash[28688]: debug 2023-10-09T05:53:07.693+0000 7fef66301680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:53:08.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_4 2023-10-09T05:53:09.115 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[27817]: cluster 2023-10-09T05:53:07.852539+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.115 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:08 smithi155 bash[28688]: debug 2023-10-09T05:53:08.849+0000 7fef66301680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:53:09.115 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.013+0000 7fef66301680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:53:09.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:09 smithi125 bash[27513]: cluster 2023-10-09T05:53:07.852539+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:09 smithi125 bash[33919]: cluster 2023-10-09T05:53:07.852539+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.392 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.113+0000 7fef66301680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:53:09.392 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.221+0000 7fef66301680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:53:09.825 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.441+0000 7fef66301680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:53:10.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.821+0000 7fef66301680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:53:10.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:09 smithi155 bash[28688]: debug 2023-10-09T05:53:09.941+0000 7fef66301680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:53:11.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.829939+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.831067+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.832116+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.839450+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: cephadm 2023-10-09T05:53:09.842401+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.842858+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.844098+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: audit 2023-10-09T05:53:09.845213+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: cephadm 2023-10-09T05:53:09.846249+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi125 2023-10-09T05:53:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[27513]: cluster 2023-10-09T05:53:09.852899+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.829939+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.831067+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.832116+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.839450+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: cephadm 2023-10-09T05:53:09.842401+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.842858+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.844098+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: audit 2023-10-09T05:53:09.845213+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: cephadm 2023-10-09T05:53:09.846249+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi125 2023-10-09T05:53:11.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:10 smithi125 bash[33919]: cluster 2023-10-09T05:53:09.852899+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[28688]: debug 2023-10-09T05:53:10.801+0000 7fef66301680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:53:11.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[28688]: debug 2023-10-09T05:53:10.909+0000 7fef66301680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:53:11.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:11 smithi155 bash[28688]: debug 2023-10-09T05:53:11.021+0000 7fef66301680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.829939+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.831067+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.832116+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.839450+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: cephadm 2023-10-09T05:53:09.842401+0000 mgr.y (mgr.14146) 55 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.842858+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.844098+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: audit 2023-10-09T05:53:09.845213+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: cephadm 2023-10-09T05:53:09.846249+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring daemon mgr.y on smithi125 2023-10-09T05:53:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:10 smithi155 bash[27817]: cluster 2023-10-09T05:53:09.852899+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.531 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:11 smithi155 bash[28688]: debug 2023-10-09T05:53:11.241+0000 7fef66301680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:53:11.864 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:11 smithi155 bash[28688]: debug 2023-10-09T05:53:11.529+0000 7fef66301680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:53:11.865 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:11 smithi155 bash[28688]: debug 2023-10-09T05:53:11.693+0000 7fef66301680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:53:12.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:11 smithi155 bash[28688]: debug 2023-10-09T05:53:11.861+0000 7fef66301680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:53:12.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[28688]: debug 2023-10-09T05:53:12.077+0000 7fef66301680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:53:12.642 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[28688]: debug 2023-10-09T05:53:12.181+0000 7fef66301680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: cluster 2023-10-09T05:53:11.853228+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: cluster 2023-10-09T05:53:12.204642+0000 mon.a (mon.0) 220 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.207806+0000 mon.c (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.208640+0000 mon.c (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.210566+0000 mon.c (mon.2) 6 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.211094+0000 mon.c (mon.2) 7 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.249551+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.252485+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.254915+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.255606+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.607708+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.609638+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.612273+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.613433+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[27513]: audit 2023-10-09T05:53:12.621072+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:13.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: cluster 2023-10-09T05:53:11.853228+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: cluster 2023-10-09T05:53:12.204642+0000 mon.a (mon.0) 220 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.207806+0000 mon.c (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.208640+0000 mon.c (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.210566+0000 mon.c (mon.2) 6 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.211094+0000 mon.c (mon.2) 7 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.249551+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.252485+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.254915+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.255606+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.607708+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:13.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.609638+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:13.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.612273+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.613433+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:13.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:12 smithi125 bash[33919]: audit 2023-10-09T05:53:12.621072+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:13.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: cluster 2023-10-09T05:53:11.853228+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: cluster 2023-10-09T05:53:12.204642+0000 mon.a (mon.0) 220 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:53:13.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.207806+0000 mon.c (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:13.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.208640+0000 mon.c (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.210566+0000 mon.c (mon.2) 6 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.211094+0000 mon.c (mon.2) 7 : audit [DBG] from='mgr.? 172.21.15.155:0/3080866404' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.249551+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.252485+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.254915+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:13.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.255606+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.607708+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:13.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.609638+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:13.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.612273+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:13.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.613433+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:13.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:12 smithi155 bash[27817]: audit 2023-10-09T05:53:12.621072+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:14.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:13 smithi125 bash[27513]: cluster 2023-10-09T05:53:12.921895+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T05:53:14.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:13 smithi125 bash[27513]: audit 2023-10-09T05:53:12.922077+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:14.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:13 smithi125 bash[33919]: cluster 2023-10-09T05:53:12.921895+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T05:53:14.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:13 smithi125 bash[33919]: audit 2023-10-09T05:53:12.922077+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:14.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:13 smithi155 bash[27817]: cluster 2023-10-09T05:53:12.921895+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 77s), standbys: x 2023-10-09T05:53:14.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:13 smithi155 bash[27817]: audit 2023-10-09T05:53:12.922077+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:15.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:14 smithi125 bash[27513]: cluster 2023-10-09T05:53:13.853665+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:15.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:14 smithi125 bash[33919]: cluster 2023-10-09T05:53:13.853665+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:15.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:14 smithi155 bash[27817]: cluster 2023-10-09T05:53:13.853665+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:17 smithi125 bash[27513]: cluster 2023-10-09T05:53:15.854116+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:17 smithi125 bash[33919]: cluster 2023-10-09T05:53:15.854116+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:17 smithi155 bash[27817]: cluster 2023-10-09T05:53:15.854116+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:19.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:19 smithi125 bash[27513]: cluster 2023-10-09T05:53:17.854590+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:19 smithi125 bash[33919]: cluster 2023-10-09T05:53:17.854590+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:19.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:19 smithi155 bash[27817]: cluster 2023-10-09T05:53:17.854590+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[27513]: cluster 2023-10-09T05:53:19.854961+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[27513]: audit 2023-10-09T05:53:20.511952+0000 mon.a (mon.0) 231 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]: dispatch 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[27513]: audit 2023-10-09T05:53:20.519118+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]': finished 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[27513]: cluster 2023-10-09T05:53:20.519185+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[27513]: audit 2023-10-09T05:53:20.519455+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[33919]: cluster 2023-10-09T05:53:19.854961+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[33919]: audit 2023-10-09T05:53:20.511952+0000 mon.a (mon.0) 231 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]: dispatch 2023-10-09T05:53:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[33919]: audit 2023-10-09T05:53:20.519118+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]': finished 2023-10-09T05:53:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[33919]: cluster 2023-10-09T05:53:20.519185+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:21 smithi125 bash[33919]: audit 2023-10-09T05:53:20.519455+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:21 smithi155 bash[27817]: cluster 2023-10-09T05:53:19.854961+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:21 smithi155 bash[27817]: audit 2023-10-09T05:53:20.511952+0000 mon.a (mon.0) 231 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]: dispatch 2023-10-09T05:53:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:21 smithi155 bash[27817]: audit 2023-10-09T05:53:20.519118+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.125:0/1010618677' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f01d5c9b-8b14-4090-a4f3-7203497e2ebe"}]': finished 2023-10-09T05:53:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:21 smithi155 bash[27817]: cluster 2023-10-09T05:53:20.519185+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:21 smithi155 bash[27817]: audit 2023-10-09T05:53:20.519455+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:22.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:22 smithi125 bash[27513]: audit 2023-10-09T05:53:21.239493+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.125:0/92890374' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:22.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:22 smithi125 bash[33919]: audit 2023-10-09T05:53:21.239493+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.125:0/92890374' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:22.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:22 smithi155 bash[27817]: audit 2023-10-09T05:53:21.239493+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.125:0/92890374' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:23.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:23 smithi125 bash[27513]: cluster 2023-10-09T05:53:21.855320+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:23.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:23 smithi125 bash[33919]: cluster 2023-10-09T05:53:21.855320+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:23.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:23 smithi155 bash[27817]: cluster 2023-10-09T05:53:21.855320+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:25 smithi125 bash[27513]: cluster 2023-10-09T05:53:23.855751+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:25 smithi125 bash[33919]: cluster 2023-10-09T05:53:23.855751+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:25 smithi155 bash[27817]: cluster 2023-10-09T05:53:23.855751+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:27.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:27 smithi125 bash[27513]: cluster 2023-10-09T05:53:25.856183+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:27.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:27 smithi125 bash[33919]: cluster 2023-10-09T05:53:25.856183+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:27 smithi155 bash[27817]: cluster 2023-10-09T05:53:25.856183+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:29 smithi125 bash[27513]: cluster 2023-10-09T05:53:27.856623+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:29 smithi125 bash[33919]: cluster 2023-10-09T05:53:27.856623+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:29.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:29 smithi155 bash[27817]: cluster 2023-10-09T05:53:27.856623+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:31.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:31 smithi125 bash[27513]: cluster 2023-10-09T05:53:29.856883+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:31.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:31 smithi125 bash[33919]: cluster 2023-10-09T05:53:29.856883+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:31 smithi155 bash[27817]: cluster 2023-10-09T05:53:29.856883+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:33.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[27513]: cluster 2023-10-09T05:53:31.857204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[27513]: audit 2023-10-09T05:53:33.155711+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[27513]: audit 2023-10-09T05:53:33.157386+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[33919]: cluster 2023-10-09T05:53:31.857204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[33919]: audit 2023-10-09T05:53:33.155711+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:33 smithi125 bash[33919]: audit 2023-10-09T05:53:33.157386+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:33 smithi155 bash[27817]: cluster 2023-10-09T05:53:31.857204+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:33 smithi155 bash[27817]: audit 2023-10-09T05:53:33.155711+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:33 smithi155 bash[27817]: audit 2023-10-09T05:53:33.157386+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:34.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:34 smithi125 bash[27513]: cephadm 2023-10-09T05:53:33.158808+0000 mgr.y (mgr.14146) 70 : cephadm [INF] Deploying daemon osd.0 on smithi125 2023-10-09T05:53:34.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:34 smithi125 bash[33919]: cephadm 2023-10-09T05:53:33.158808+0000 mgr.y (mgr.14146) 70 : cephadm [INF] Deploying daemon osd.0 on smithi125 2023-10-09T05:53:34.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:34 smithi155 bash[27817]: cephadm 2023-10-09T05:53:33.158808+0000 mgr.y (mgr.14146) 70 : cephadm [INF] Deploying daemon osd.0 on smithi125 2023-10-09T05:53:35.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:35 smithi125 bash[27513]: cluster 2023-10-09T05:53:33.857703+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:35.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:35 smithi125 bash[33919]: cluster 2023-10-09T05:53:33.857703+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:35.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:35 smithi155 bash[27817]: cluster 2023-10-09T05:53:33.857703+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:36.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:36 smithi125 bash[27513]: cluster 2023-10-09T05:53:35.858141+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:36.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:36 smithi125 bash[33919]: cluster 2023-10-09T05:53:35.858141+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:36.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:36 smithi155 bash[27817]: cluster 2023-10-09T05:53:35.858141+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:39.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[33919]: audit 2023-10-09T05:53:37.791030+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:39.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[33919]: audit 2023-10-09T05:53:37.798524+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:39.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[33919]: cluster 2023-10-09T05:53:37.858527+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:39.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[27513]: audit 2023-10-09T05:53:37.791030+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:39.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[27513]: audit 2023-10-09T05:53:37.798524+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:39.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:38 smithi125 bash[27513]: cluster 2023-10-09T05:53:37.858527+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:39.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:38 smithi155 bash[27817]: audit 2023-10-09T05:53:37.791030+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:39.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:38 smithi155 bash[27817]: audit 2023-10-09T05:53:37.798524+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:39.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:38 smithi155 bash[27817]: cluster 2023-10-09T05:53:37.858527+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:41.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:41 smithi125 bash[27513]: cluster 2023-10-09T05:53:39.858935+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:41.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:41 smithi125 bash[33919]: cluster 2023-10-09T05:53:39.858935+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:41.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:41 smithi155 bash[27817]: cluster 2023-10-09T05:53:39.858935+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:43.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[27513]: cluster 2023-10-09T05:53:41.859364+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:43.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[27513]: audit 2023-10-09T05:53:42.893519+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:43.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[27513]: audit 2023-10-09T05:53:43.179367+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:43.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[27513]: audit 2023-10-09T05:53:43.180708+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[27513]: audit 2023-10-09T05:53:43.187829+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[33919]: cluster 2023-10-09T05:53:41.859364+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[33919]: audit 2023-10-09T05:53:42.893519+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[33919]: audit 2023-10-09T05:53:43.179367+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[33919]: audit 2023-10-09T05:53:43.180708+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:43.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:43 smithi125 bash[33919]: audit 2023-10-09T05:53:43.187829+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:43.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:43 smithi155 bash[27817]: cluster 2023-10-09T05:53:41.859364+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:43.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:43 smithi155 bash[27817]: audit 2023-10-09T05:53:42.893519+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:43.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:43 smithi155 bash[27817]: audit 2023-10-09T05:53:43.179367+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:43.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:43 smithi155 bash[27817]: audit 2023-10-09T05:53:43.180708+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:43.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:43 smithi155 bash[27817]: audit 2023-10-09T05:53:43.187829+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:45.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:45 smithi125 bash[27513]: cluster 2023-10-09T05:53:43.859653+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:45.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:45 smithi125 bash[27513]: audit 2023-10-09T05:53:44.478642+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:45.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:45 smithi125 bash[33919]: cluster 2023-10-09T05:53:43.859653+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:45.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:45 smithi125 bash[33919]: audit 2023-10-09T05:53:44.478642+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:45.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:45 smithi155 bash[27817]: cluster 2023-10-09T05:53:43.859653+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:45.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:45 smithi155 bash[27817]: audit 2023-10-09T05:53:44.478642+0000 mon.a (mon.0) 244 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:46.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[33919]: audit 2023-10-09T05:53:45.242897+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:46.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[33919]: cluster 2023-10-09T05:53:45.243026+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:46.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[33919]: audit 2023-10-09T05:53:45.243248+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:46.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[33919]: audit 2023-10-09T05:53:45.243720+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:53:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[27513]: audit 2023-10-09T05:53:45.242897+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:46.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[27513]: cluster 2023-10-09T05:53:45.243026+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[27513]: audit 2023-10-09T05:53:45.243248+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:46.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:46 smithi125 bash[27513]: audit 2023-10-09T05:53:45.243720+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:53:46.551 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 0 on host 'smithi125' 2023-10-09T05:53:46.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:46 smithi155 bash[27817]: audit 2023-10-09T05:53:45.242897+0000 mon.a (mon.0) 245 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:46.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:46 smithi155 bash[27817]: cluster 2023-10-09T05:53:45.243026+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:46.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:46 smithi155 bash[27817]: audit 2023-10-09T05:53:45.243248+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:46.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:46 smithi155 bash[27817]: audit 2023-10-09T05:53:45.243720+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:53:47.320 DEBUG:teuthology.orchestra.run.smithi125:osd.0> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T05:53:47.322 INFO:tasks.cephadm:Deploying osd.1 on smithi125 with /dev/vg_nvme/lv_3... 2023-10-09T05:53:47.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:53:47.423 INFO:journalctl@ceph.osd.0.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:53:47.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: cluster 2023-10-09T05:53:45.859927+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:47.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: audit 2023-10-09T05:53:46.244481+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: cluster 2023-10-09T05:53:46.244535+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: audit 2023-10-09T05:53:46.245128+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: audit 2023-10-09T05:53:46.253745+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: audit 2023-10-09T05:53:46.549926+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[27513]: audit 2023-10-09T05:53:46.562378+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:47.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: cluster 2023-10-09T05:53:45.859927+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: audit 2023-10-09T05:53:46.244481+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: cluster 2023-10-09T05:53:46.244535+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: audit 2023-10-09T05:53:46.245128+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: audit 2023-10-09T05:53:46.253745+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: audit 2023-10-09T05:53:46.549926+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:47.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:47 smithi125 bash[33919]: audit 2023-10-09T05:53:46.562378+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:47.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: cluster 2023-10-09T05:53:45.859927+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: audit 2023-10-09T05:53:46.244481+0000 mon.a (mon.0) 249 : audit [INF] from='osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: cluster 2023-10-09T05:53:46.244535+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: audit 2023-10-09T05:53:46.245128+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: audit 2023-10-09T05:53:46.253745+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: audit 2023-10-09T05:53:46.549926+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:47 smithi155 bash[27817]: audit 2023-10-09T05:53:46.562378+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:48.582 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[27513]: cluster 2023-10-09T05:53:45.508315+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[27513]: cluster 2023-10-09T05:53:45.508380+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[27513]: cluster 2023-10-09T05:53:47.255056+0000 mon.a (mon.0) 255 : cluster [INF] osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200] boot 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[27513]: cluster 2023-10-09T05:53:47.255129+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[27513]: audit 2023-10-09T05:53:47.255735+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[33919]: cluster 2023-10-09T05:53:45.508315+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[33919]: cluster 2023-10-09T05:53:45.508380+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[33919]: cluster 2023-10-09T05:53:47.255056+0000 mon.a (mon.0) 255 : cluster [INF] osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200] boot 2023-10-09T05:53:48.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[33919]: cluster 2023-10-09T05:53:47.255129+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:48.584 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:48 smithi125 bash[33919]: audit 2023-10-09T05:53:47.255735+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:48.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:48 smithi155 bash[27817]: cluster 2023-10-09T05:53:45.508315+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:53:48.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:48 smithi155 bash[27817]: cluster 2023-10-09T05:53:45.508380+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:53:48.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:48 smithi155 bash[27817]: cluster 2023-10-09T05:53:47.255056+0000 mon.a (mon.0) 255 : cluster [INF] osd.0 [v2:172.21.15.125:6802/3657277200,v1:172.21.15.125:6803/3657277200] boot 2023-10-09T05:53:48.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:48 smithi155 bash[27817]: cluster 2023-10-09T05:53:47.255129+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:48.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:48 smithi155 bash[27817]: audit 2023-10-09T05:53:47.255735+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:49.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:49 smithi125 bash[27513]: cluster 2023-10-09T05:53:47.860318+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:49 smithi125 bash[27513]: cluster 2023-10-09T05:53:48.268356+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:49.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:49 smithi125 bash[33919]: cluster 2023-10-09T05:53:47.860318+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:49.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:49 smithi125 bash[33919]: cluster 2023-10-09T05:53:48.268356+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:49 smithi155 bash[27817]: cluster 2023-10-09T05:53:47.860318+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:49 smithi155 bash[27817]: cluster 2023-10-09T05:53:48.268356+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:50.918 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:53:50.918 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:53:50.918 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T05:53:50.918 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T05:53:50.918 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155508 s, 674 MB/s 2023-10-09T05:53:50.919 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T05:53:51.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:51 smithi125 bash[27513]: cluster 2023-10-09T05:53:49.860814+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:51 smithi125 bash[33919]: cluster 2023-10-09T05:53:49.860814+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:51 smithi155 bash[27817]: cluster 2023-10-09T05:53:49.860814+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_3 2023-10-09T05:53:52.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:52 smithi125 bash[27513]: cluster 2023-10-09T05:53:51.861141+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:52.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:52 smithi125 bash[33919]: cluster 2023-10-09T05:53:51.861141+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:52.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:52 smithi155 bash[27817]: cluster 2023-10-09T05:53:51.861141+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:54 smithi125 bash[27513]: cluster 2023-10-09T05:53:53.861607+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:54 smithi125 bash[33919]: cluster 2023-10-09T05:53:53.861607+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:54 smithi155 bash[27817]: cluster 2023-10-09T05:53:53.861607+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:57.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[27513]: cluster 2023-10-09T05:53:55.862141+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[27513]: audit 2023-10-09T05:53:56.663311+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[27513]: audit 2023-10-09T05:53:56.668070+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[27513]: audit 2023-10-09T05:53:56.669276+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[33919]: cluster 2023-10-09T05:53:55.862141+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[33919]: audit 2023-10-09T05:53:56.663311+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[33919]: audit 2023-10-09T05:53:56.668070+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:57.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:57 smithi125 bash[33919]: audit 2023-10-09T05:53:56.669276+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:57 smithi155 bash[27817]: cluster 2023-10-09T05:53:55.862141+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:57 smithi155 bash[27817]: audit 2023-10-09T05:53:56.663311+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:57 smithi155 bash[27817]: audit 2023-10-09T05:53:56.668070+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:57 smithi155 bash[27817]: audit 2023-10-09T05:53:56.669276+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:58.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:56.657062+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.016292+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.017563+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.023855+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.025280+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.026279+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[27513]: audit 2023-10-09T05:53:58.032165+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:56.657062+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.016292+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.017563+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.023855+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.025280+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.026279+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:58 smithi125 bash[33919]: audit 2023-10-09T05:53:58.032165+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:56.657062+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.016292+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.017563+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.023855+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.025280+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.026279+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:58.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:58 smithi155 bash[27817]: audit 2023-10-09T05:53:58.032165+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:53:59.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[27513]: cluster 2023-10-09T05:53:57.862683+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:59.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[27513]: cephadm 2023-10-09T05:53:58.009534+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:53:59.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[27513]: cephadm 2023-10-09T05:53:58.018481+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 16241M 2023-10-09T05:53:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[33919]: cluster 2023-10-09T05:53:57.862683+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[33919]: cephadm 2023-10-09T05:53:58.009534+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:53:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:53:59 smithi125 bash[33919]: cephadm 2023-10-09T05:53:58.018481+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 16241M 2023-10-09T05:53:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:59 smithi155 bash[27817]: cluster 2023-10-09T05:53:57.862683+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:59 smithi155 bash[27817]: cephadm 2023-10-09T05:53:58.009534+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:53:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:53:59 smithi155 bash[27817]: cephadm 2023-10-09T05:53:58.018481+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 16241M 2023-10-09T05:54:01.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:01 smithi125 bash[33919]: cluster 2023-10-09T05:53:59.863156+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:01 smithi125 bash[27513]: cluster 2023-10-09T05:53:59.863156+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:01 smithi155 bash[27817]: cluster 2023-10-09T05:53:59.863156+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:03.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:03 smithi125 bash[27513]: cluster 2023-10-09T05:54:01.863660+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:03.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:03 smithi125 bash[33919]: cluster 2023-10-09T05:54:01.863660+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:03.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:03 smithi155 bash[27817]: cluster 2023-10-09T05:54:01.863660+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:05.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[27513]: cluster 2023-10-09T05:54:03.864107+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[27513]: audit 2023-10-09T05:54:04.747469+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]: dispatch 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[27513]: audit 2023-10-09T05:54:04.754718+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]': finished 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[27513]: cluster 2023-10-09T05:54:04.754859+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[27513]: audit 2023-10-09T05:54:04.755103+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[33919]: cluster 2023-10-09T05:54:03.864107+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[33919]: audit 2023-10-09T05:54:04.747469+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]: dispatch 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[33919]: audit 2023-10-09T05:54:04.754718+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]': finished 2023-10-09T05:54:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[33919]: cluster 2023-10-09T05:54:04.754859+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:54:05.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:05 smithi125 bash[33919]: audit 2023-10-09T05:54:04.755103+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:05 smithi155 bash[27817]: cluster 2023-10-09T05:54:03.864107+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:05 smithi155 bash[27817]: audit 2023-10-09T05:54:04.747469+0000 mon.a (mon.0) 268 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]: dispatch 2023-10-09T05:54:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:05 smithi155 bash[27817]: audit 2023-10-09T05:54:04.754718+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.125:0/4190724610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b751bb-a1ce-4054-993d-edd9b0632cc2"}]': finished 2023-10-09T05:54:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:05 smithi155 bash[27817]: cluster 2023-10-09T05:54:04.754859+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:54:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:05 smithi155 bash[27817]: audit 2023-10-09T05:54:04.755103+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:06.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:06 smithi125 bash[33919]: audit 2023-10-09T05:54:05.482209+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.125:0/2731560694' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:06.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:06 smithi125 bash[27513]: audit 2023-10-09T05:54:05.482209+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.125:0/2731560694' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:06.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:06 smithi155 bash[27817]: audit 2023-10-09T05:54:05.482209+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.125:0/2731560694' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:07.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:07 smithi125 bash[27513]: cluster 2023-10-09T05:54:05.864555+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:07.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:07 smithi125 bash[33919]: cluster 2023-10-09T05:54:05.864555+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:07.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:07 smithi155 bash[27817]: cluster 2023-10-09T05:54:05.864555+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:09.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:09 smithi125 bash[27513]: cluster 2023-10-09T05:54:07.865014+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:09.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:09 smithi125 bash[33919]: cluster 2023-10-09T05:54:07.865014+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:09.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:09 smithi155 bash[27817]: cluster 2023-10-09T05:54:07.865014+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:11.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:11 smithi125 bash[27513]: cluster 2023-10-09T05:54:09.865519+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:11.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:11 smithi125 bash[33919]: cluster 2023-10-09T05:54:09.865519+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:11.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:11 smithi155 bash[27817]: cluster 2023-10-09T05:54:09.865519+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:12.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:12 smithi125 bash[27513]: cluster 2023-10-09T05:54:11.865984+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:12 smithi125 bash[33919]: cluster 2023-10-09T05:54:11.865984+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:12.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:12 smithi155 bash[27817]: cluster 2023-10-09T05:54:11.865984+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:15.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:14 smithi125 bash[27513]: cluster 2023-10-09T05:54:13.866532+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:15.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:14 smithi125 bash[33919]: cluster 2023-10-09T05:54:13.866532+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:15.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:14 smithi155 bash[27817]: cluster 2023-10-09T05:54:13.866532+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:17.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:17 smithi125 bash[27513]: cluster 2023-10-09T05:54:15.866968+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:17.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:17 smithi125 bash[33919]: cluster 2023-10-09T05:54:15.866968+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:17.780 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:17 smithi155 bash[27817]: cluster 2023-10-09T05:54:15.866968+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:18.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:18 smithi125 bash[27513]: audit 2023-10-09T05:54:17.422103+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:54:18.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:18 smithi125 bash[27513]: audit 2023-10-09T05:54:17.423658+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:18.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:18 smithi125 bash[33919]: audit 2023-10-09T05:54:17.422103+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:54:18.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:18 smithi125 bash[33919]: audit 2023-10-09T05:54:17.423658+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:18.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:18 smithi155 bash[27817]: audit 2023-10-09T05:54:17.422103+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:54:18.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:18 smithi155 bash[27817]: audit 2023-10-09T05:54:17.423658+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:19.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:19 smithi125 bash[27513]: cephadm 2023-10-09T05:54:17.424807+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi125 2023-10-09T05:54:19.526 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:19 smithi125 bash[27513]: cluster 2023-10-09T05:54:17.867437+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:19 smithi125 bash[33919]: cephadm 2023-10-09T05:54:17.424807+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi125 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:19 smithi125 bash[33919]: cluster 2023-10-09T05:54:17.867437+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:19.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:19 smithi155 bash[27817]: cephadm 2023-10-09T05:54:17.424807+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi125 2023-10-09T05:54:19.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:19 smithi155 bash[27817]: cluster 2023-10-09T05:54:17.867437+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:21.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:21 smithi155 bash[27817]: cluster 2023-10-09T05:54:19.867962+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:21.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:21 smithi125 bash[27513]: cluster 2023-10-09T05:54:19.867962+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:21.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:21 smithi125 bash[33919]: cluster 2023-10-09T05:54:19.867962+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:22.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:22 smithi125 bash[33919]: audit 2023-10-09T05:54:21.858733+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:22.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:22 smithi125 bash[33919]: audit 2023-10-09T05:54:21.863577+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:22.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:22 smithi125 bash[27513]: audit 2023-10-09T05:54:21.858733+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:22.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:22 smithi125 bash[27513]: audit 2023-10-09T05:54:21.863577+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:22.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:22 smithi155 bash[27817]: audit 2023-10-09T05:54:21.858733+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:22.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:22 smithi155 bash[27817]: audit 2023-10-09T05:54:21.863577+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:23.625 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:23 smithi125 bash[33919]: cluster 2023-10-09T05:54:21.868262+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:23.625 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:23 smithi125 bash[27513]: cluster 2023-10-09T05:54:21.868262+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:23.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:23 smithi155 bash[27817]: cluster 2023-10-09T05:54:21.868262+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:25.565 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:25 smithi125 bash[33919]: cluster 2023-10-09T05:54:23.868571+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:25.565 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:25 smithi125 bash[27513]: cluster 2023-10-09T05:54:23.868571+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:25.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:25 smithi155 bash[27817]: cluster 2023-10-09T05:54:23.868571+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:27.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[27513]: cluster 2023-10-09T05:54:25.869138+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[27513]: audit 2023-10-09T05:54:27.044321+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[27513]: audit 2023-10-09T05:54:27.045657+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[27513]: audit 2023-10-09T05:54:27.046904+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[27513]: audit 2023-10-09T05:54:27.052332+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[33919]: cluster 2023-10-09T05:54:25.869138+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[33919]: audit 2023-10-09T05:54:27.044321+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[33919]: audit 2023-10-09T05:54:27.045657+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:27.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[33919]: audit 2023-10-09T05:54:27.046904+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:27.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:27 smithi125 bash[33919]: audit 2023-10-09T05:54:27.052332+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:27 smithi155 bash[27817]: cluster 2023-10-09T05:54:25.869138+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:27 smithi155 bash[27817]: audit 2023-10-09T05:54:27.044321+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:27 smithi155 bash[27817]: audit 2023-10-09T05:54:27.045657+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:27 smithi155 bash[27817]: audit 2023-10-09T05:54:27.046904+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:27 smithi155 bash[27817]: audit 2023-10-09T05:54:27.052332+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:29.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[27513]: cluster 2023-10-09T05:54:27.869628+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:29.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[27513]: audit 2023-10-09T05:54:28.590595+0000 mon.c (mon.2) 9 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:29.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[27513]: audit 2023-10-09T05:54:28.591124+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:29.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[33919]: cluster 2023-10-09T05:54:27.869628+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:29.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[33919]: audit 2023-10-09T05:54:28.590595+0000 mon.c (mon.2) 9 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:29.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:29 smithi125 bash[33919]: audit 2023-10-09T05:54:28.591124+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:29.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:29 smithi155 bash[27817]: cluster 2023-10-09T05:54:27.869628+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:29.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:29 smithi155 bash[27817]: audit 2023-10-09T05:54:28.590595+0000 mon.c (mon.2) 9 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:29.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:29 smithi155 bash[27817]: audit 2023-10-09T05:54:28.591124+0000 mon.a (mon.0) 280 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:54:30.546 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 1 on host 'smithi125' 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:29.270740+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: cluster 2023-10-09T05:54:29.270865+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:29.272412+0000 mon.c (mon.2) 10 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:29.272717+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:29.273214+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:30.270436+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:54:30.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: cluster 2023-10-09T05:54:30.270660+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[27513]: audit 2023-10-09T05:54:30.270834+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:29.270740+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: cluster 2023-10-09T05:54:29.270865+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:29.272412+0000 mon.c (mon.2) 10 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:29.272717+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:29.273214+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:30.270436+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:54:30.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: cluster 2023-10-09T05:54:30.270660+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:54:30.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:30 smithi125 bash[33919]: audit 2023-10-09T05:54:30.270834+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:29.270740+0000 mon.a (mon.0) 281 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:54:30.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: cluster 2023-10-09T05:54:29.270865+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:29.272412+0000 mon.c (mon.2) 10 : audit [INF] from='osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:29.272717+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:29.273214+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:30.270436+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: cluster 2023-10-09T05:54:30.270660+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:54:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:30 smithi155 bash[27817]: audit 2023-10-09T05:54:30.270834+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.240 DEBUG:teuthology.orchestra.run.smithi125:osd.1> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T05:54:31.243 INFO:tasks.cephadm:Deploying osd.2 on smithi125 with /dev/vg_nvme/lv_2... 2023-10-09T05:54:31.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:54:31.346 INFO:journalctl@ceph.osd.1.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:54:31.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[27513]: cluster 2023-10-09T05:54:29.870151+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[27513]: audit 2023-10-09T05:54:30.276816+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[27513]: audit 2023-10-09T05:54:30.544619+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[27513]: audit 2023-10-09T05:54:30.577147+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[27513]: audit 2023-10-09T05:54:31.276672+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[33919]: cluster 2023-10-09T05:54:29.870151+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[33919]: audit 2023-10-09T05:54:30.276816+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[33919]: audit 2023-10-09T05:54:30.544619+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[33919]: audit 2023-10-09T05:54:30.577147+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:31 smithi125 bash[33919]: audit 2023-10-09T05:54:31.276672+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:31 smithi155 bash[27817]: cluster 2023-10-09T05:54:29.870151+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:31 smithi155 bash[27817]: audit 2023-10-09T05:54:30.276816+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:31 smithi155 bash[27817]: audit 2023-10-09T05:54:30.544619+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:31 smithi155 bash[27817]: audit 2023-10-09T05:54:30.577147+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:31 smithi155 bash[27817]: audit 2023-10-09T05:54:31.276672+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:32.585 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: cluster 2023-10-09T05:54:29.584719+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:54:32.585 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: cluster 2023-10-09T05:54:29.584783+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: cluster 2023-10-09T05:54:31.293661+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570] boot 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: cluster 2023-10-09T05:54:31.293851+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: audit 2023-10-09T05:54:31.294277+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[27513]: cluster 2023-10-09T05:54:31.870632+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: cluster 2023-10-09T05:54:29.584719+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: cluster 2023-10-09T05:54:29.584783+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:54:32.586 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: cluster 2023-10-09T05:54:31.293661+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570] boot 2023-10-09T05:54:32.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: cluster 2023-10-09T05:54:31.293851+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:54:32.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: audit 2023-10-09T05:54:31.294277+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:32.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:32 smithi125 bash[33919]: cluster 2023-10-09T05:54:31.870632+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:32.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: cluster 2023-10-09T05:54:29.584719+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:54:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: cluster 2023-10-09T05:54:29.584783+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:54:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: cluster 2023-10-09T05:54:31.293661+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.125:6810/753846570,v1:172.21.15.125:6811/753846570] boot 2023-10-09T05:54:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: cluster 2023-10-09T05:54:31.293851+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:54:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: audit 2023-10-09T05:54:31.294277+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:54:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:32 smithi155 bash[27817]: cluster 2023-10-09T05:54:31.870632+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:54:33.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:33 smithi125 bash[27513]: cluster 2023-10-09T05:54:32.306651+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:54:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:33 smithi125 bash[33919]: cluster 2023-10-09T05:54:32.306651+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:54:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:33 smithi155 bash[27817]: cluster 2023-10-09T05:54:32.306651+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:54:34.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:34 smithi125 bash[27513]: cluster 2023-10-09T05:54:33.870953+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:34.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:34 smithi125 bash[33919]: cluster 2023-10-09T05:54:33.870953+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:34.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:34 smithi155 bash[27817]: cluster 2023-10-09T05:54:33.870953+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:34.853 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:54:34.853 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:54:34.853 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T05:54:34.853 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T05:54:34.853 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0210261 s, 499 MB/s 2023-10-09T05:54:34.854 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T05:54:35.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_2 2023-10-09T05:54:37.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:37 smithi125 bash[27513]: cluster 2023-10-09T05:54:35.871345+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:37.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:37 smithi125 bash[33919]: cluster 2023-10-09T05:54:35.871345+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:37.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:37 smithi155 bash[27817]: cluster 2023-10-09T05:54:35.871345+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:39.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:39 smithi125 bash[27513]: cluster 2023-10-09T05:54:37.871787+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:39.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:39 smithi125 bash[33919]: cluster 2023-10-09T05:54:37.871787+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:39.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:39 smithi155 bash[27817]: cluster 2023-10-09T05:54:37.871787+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[33919]: cluster 2023-10-09T05:54:39.872098+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[33919]: audit 2023-10-09T05:54:40.253554+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[33919]: audit 2023-10-09T05:54:40.257609+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[33919]: audit 2023-10-09T05:54:40.260581+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[33919]: audit 2023-10-09T05:54:40.261600+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:41.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[27513]: cluster 2023-10-09T05:54:39.872098+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:41.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[27513]: audit 2023-10-09T05:54:40.253554+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:41.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[27513]: audit 2023-10-09T05:54:40.257609+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:41.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[27513]: audit 2023-10-09T05:54:40.260581+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:41.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:41 smithi125 bash[27513]: audit 2023-10-09T05:54:40.261600+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:41.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:41 smithi155 bash[27817]: cluster 2023-10-09T05:54:39.872098+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:41 smithi155 bash[27817]: audit 2023-10-09T05:54:40.253554+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:41 smithi155 bash[27817]: audit 2023-10-09T05:54:40.257609+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:41 smithi155 bash[27817]: audit 2023-10-09T05:54:40.260581+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:41 smithi155 bash[27817]: audit 2023-10-09T05:54:40.261600+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:43.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: cephadm 2023-10-09T05:54:41.815980+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.824139+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.825599+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.826640+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: cephadm 2023-10-09T05:54:41.827520+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 8120M 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.833674+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.835227+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.836193+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: audit 2023-10-09T05:54:41.842655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[33919]: cluster 2023-10-09T05:54:41.872548+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: cephadm 2023-10-09T05:54:41.815980+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.824139+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.825599+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.826640+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: cephadm 2023-10-09T05:54:41.827520+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 8120M 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.833674+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.835227+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.836193+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:43.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: audit 2023-10-09T05:54:41.842655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:42 smithi125 bash[27513]: cluster 2023-10-09T05:54:41.872548+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: cephadm 2023-10-09T05:54:41.815980+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:54:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.824139+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.825599+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.826640+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: cephadm 2023-10-09T05:54:41.827520+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 8120M 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.833674+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.835227+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.836193+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: audit 2023-10-09T05:54:41.842655+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:54:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:42 smithi155 bash[27817]: cluster 2023-10-09T05:54:41.872548+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:45.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:44 smithi125 bash[33919]: cluster 2023-10-09T05:54:43.873077+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:45.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:44 smithi125 bash[27513]: cluster 2023-10-09T05:54:43.873077+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:45.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:44 smithi155 bash[27817]: cluster 2023-10-09T05:54:43.873077+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:47.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:47 smithi125 bash[27513]: cluster 2023-10-09T05:54:45.873555+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:47.624 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:47 smithi125 bash[33919]: cluster 2023-10-09T05:54:45.873555+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:47.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:47 smithi155 bash[27817]: cluster 2023-10-09T05:54:45.873555+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:49.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: cluster 2023-10-09T05:54:47.873853+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: audit 2023-10-09T05:54:48.527515+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.125:0/968488027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: audit 2023-10-09T05:54:48.528166+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: audit 2023-10-09T05:54:48.539830+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]': finished 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: cluster 2023-10-09T05:54:48.539958+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: audit 2023-10-09T05:54:48.540177+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[27513]: audit 2023-10-09T05:54:49.235342+0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.125:0/1905492660' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: cluster 2023-10-09T05:54:47.873853+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: audit 2023-10-09T05:54:48.527515+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.125:0/968488027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: audit 2023-10-09T05:54:48.528166+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: audit 2023-10-09T05:54:48.539830+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]': finished 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: cluster 2023-10-09T05:54:48.539958+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: audit 2023-10-09T05:54:48.540177+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:49 smithi125 bash[33919]: audit 2023-10-09T05:54:49.235342+0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.125:0/1905492660' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: cluster 2023-10-09T05:54:47.873853+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: audit 2023-10-09T05:54:48.527515+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.125:0/968488027' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: audit 2023-10-09T05:54:48.528166+0000 mon.a (mon.0) 306 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]: dispatch 2023-10-09T05:54:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: audit 2023-10-09T05:54:48.539830+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc437638-50e5-4487-9372-5300b0e63955"}]': finished 2023-10-09T05:54:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: cluster 2023-10-09T05:54:48.539958+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: audit 2023-10-09T05:54:48.540177+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:49 smithi155 bash[27817]: audit 2023-10-09T05:54:49.235342+0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.125:0/1905492660' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:51.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:51 smithi125 bash[33919]: cluster 2023-10-09T05:54:49.874199+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:51.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:51 smithi125 bash[27513]: cluster 2023-10-09T05:54:49.874199+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:51.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:51 smithi155 bash[27817]: cluster 2023-10-09T05:54:49.874199+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:53.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:53 smithi125 bash[27513]: cluster 2023-10-09T05:54:51.874528+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:53.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:53 smithi125 bash[33919]: cluster 2023-10-09T05:54:51.874528+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:53 smithi155 bash[27817]: cluster 2023-10-09T05:54:51.874528+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:55.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:55 smithi125 bash[33919]: cluster 2023-10-09T05:54:53.875119+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:55.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:55 smithi125 bash[27513]: cluster 2023-10-09T05:54:53.875119+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:55.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:55 smithi155 bash[27817]: cluster 2023-10-09T05:54:53.875119+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:57.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:57 smithi125 bash[33919]: cluster 2023-10-09T05:54:55.875712+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:57.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:57 smithi125 bash[27513]: cluster 2023-10-09T05:54:55.875712+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:57 smithi155 bash[27817]: cluster 2023-10-09T05:54:55.875712+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:59.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:54:59 smithi125 bash[33919]: cluster 2023-10-09T05:54:57.876229+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:59.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:54:59 smithi125 bash[27513]: cluster 2023-10-09T05:54:57.876229+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:54:59 smithi155 bash[27817]: cluster 2023-10-09T05:54:57.876229+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[33919]: cluster 2023-10-09T05:54:59.876526+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[33919]: audit 2023-10-09T05:55:00.957655+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[33919]: audit 2023-10-09T05:55:00.959250+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[27513]: cluster 2023-10-09T05:54:59.876526+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[27513]: audit 2023-10-09T05:55:00.957655+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:55:01.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:01 smithi125 bash[27513]: audit 2023-10-09T05:55:00.959250+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:01 smithi155 bash[27817]: cluster 2023-10-09T05:54:59.876526+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:01 smithi155 bash[27817]: audit 2023-10-09T05:55:00.957655+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:55:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:01 smithi155 bash[27817]: audit 2023-10-09T05:55:00.959250+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:02.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:02 smithi125 bash[27513]: cephadm 2023-10-09T05:55:00.960526+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi125 2023-10-09T05:55:02.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:02 smithi125 bash[33919]: cephadm 2023-10-09T05:55:00.960526+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi125 2023-10-09T05:55:02.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:02 smithi155 bash[27817]: cephadm 2023-10-09T05:55:00.960526+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi125 2023-10-09T05:55:03.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:03 smithi125 bash[33919]: cluster 2023-10-09T05:55:01.876911+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:03.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:03 smithi125 bash[27513]: cluster 2023-10-09T05:55:01.876911+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:03.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:03 smithi155 bash[27817]: cluster 2023-10-09T05:55:01.876911+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:04.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:04 smithi125 bash[27513]: cluster 2023-10-09T05:55:03.877364+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:04.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:04 smithi125 bash[33919]: cluster 2023-10-09T05:55:03.877364+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:04.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:04 smithi155 bash[27817]: cluster 2023-10-09T05:55:03.877364+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:06.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:06 smithi125 bash[27513]: audit 2023-10-09T05:55:05.226612+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:06.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:06 smithi125 bash[27513]: audit 2023-10-09T05:55:05.268356+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:06.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:06 smithi125 bash[33919]: audit 2023-10-09T05:55:05.226612+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:06.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:06 smithi125 bash[33919]: audit 2023-10-09T05:55:05.268356+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:06.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:06 smithi155 bash[27817]: audit 2023-10-09T05:55:05.226612+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:06.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:06 smithi155 bash[27817]: audit 2023-10-09T05:55:05.268356+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:07.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:07 smithi125 bash[33919]: cluster 2023-10-09T05:55:05.877838+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:07.522 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:07 smithi125 bash[27513]: cluster 2023-10-09T05:55:05.877838+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:07.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:07 smithi155 bash[27817]: cluster 2023-10-09T05:55:05.877838+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:09.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:09 smithi125 bash[27513]: cluster 2023-10-09T05:55:07.878278+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:09.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:09 smithi125 bash[33919]: cluster 2023-10-09T05:55:07.878278+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:09.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:09 smithi155 bash[27817]: cluster 2023-10-09T05:55:07.878278+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:11.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:11 smithi125 bash[27513]: cluster 2023-10-09T05:55:09.878778+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:11.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:11 smithi125 bash[33919]: cluster 2023-10-09T05:55:09.878778+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:11.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:11 smithi155 bash[27817]: cluster 2023-10-09T05:55:09.878778+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:12.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: audit 2023-10-09T05:55:11.422553+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: audit 2023-10-09T05:55:11.694623+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: audit 2023-10-09T05:55:11.696045+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: audit 2023-10-09T05:55:11.702354+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: cluster 2023-10-09T05:55:11.879285+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[27513]: audit 2023-10-09T05:55:11.976027+0000 mon.a (mon.0) 319 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: audit 2023-10-09T05:55:11.422553+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: audit 2023-10-09T05:55:11.694623+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: audit 2023-10-09T05:55:11.696045+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:12.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: audit 2023-10-09T05:55:11.702354+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: cluster 2023-10-09T05:55:11.879285+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:12.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:12 smithi125 bash[33919]: audit 2023-10-09T05:55:11.976027+0000 mon.a (mon.0) 319 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:55:12.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: audit 2023-10-09T05:55:11.422553+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: audit 2023-10-09T05:55:11.694623+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: audit 2023-10-09T05:55:11.696045+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: audit 2023-10-09T05:55:11.702354+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: cluster 2023-10-09T05:55:11.879285+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:12 smithi155 bash[27817]: audit 2023-10-09T05:55:11.976027+0000 mon.a (mon.0) 319 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:55:14.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[27513]: audit 2023-10-09T05:55:12.711144+0000 mon.a (mon.0) 320 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[27513]: cluster 2023-10-09T05:55:12.711241+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[27513]: audit 2023-10-09T05:55:12.711536+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[27513]: audit 2023-10-09T05:55:12.712640+0000 mon.a (mon.0) 323 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[33919]: audit 2023-10-09T05:55:12.711144+0000 mon.a (mon.0) 320 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[33919]: cluster 2023-10-09T05:55:12.711241+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[33919]: audit 2023-10-09T05:55:12.711536+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:13 smithi125 bash[33919]: audit 2023-10-09T05:55:12.712640+0000 mon.a (mon.0) 323 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:13 smithi155 bash[27817]: audit 2023-10-09T05:55:12.711144+0000 mon.a (mon.0) 320 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:55:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:13 smithi155 bash[27817]: cluster 2023-10-09T05:55:12.711241+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:55:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:13 smithi155 bash[27817]: audit 2023-10-09T05:55:12.711536+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:13 smithi155 bash[27817]: audit 2023-10-09T05:55:12.712640+0000 mon.a (mon.0) 323 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:14.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[33919]: audit 2023-10-09T05:55:13.714013+0000 mon.a (mon.0) 324 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:55:14.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[33919]: cluster 2023-10-09T05:55:13.714120+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:55:14.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[33919]: audit 2023-10-09T05:55:13.714803+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[33919]: audit 2023-10-09T05:55:13.721209+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[33919]: cluster 2023-10-09T05:55:13.879734+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[27513]: audit 2023-10-09T05:55:13.714013+0000 mon.a (mon.0) 324 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[27513]: cluster 2023-10-09T05:55:13.714120+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[27513]: audit 2023-10-09T05:55:13.714803+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[27513]: audit 2023-10-09T05:55:13.721209+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:14.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:14 smithi125 bash[27513]: cluster 2023-10-09T05:55:13.879734+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:15.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:14 smithi155 bash[27817]: audit 2023-10-09T05:55:13.714013+0000 mon.a (mon.0) 324 : audit [INF] from='osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:55:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:14 smithi155 bash[27817]: cluster 2023-10-09T05:55:13.714120+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:55:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:14 smithi155 bash[27817]: audit 2023-10-09T05:55:13.714803+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:14 smithi155 bash[27817]: audit 2023-10-09T05:55:13.721209+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:14 smithi155 bash[27817]: cluster 2023-10-09T05:55:13.879734+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:55:15.209 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 2 on host 'smithi125' 2023-10-09T05:55:15.925 DEBUG:teuthology.orchestra.run.smithi125:osd.2> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T05:55:15.928 INFO:tasks.cephadm:Deploying osd.3 on smithi125 with /dev/vg_nvme/lv_1... 2023-10-09T05:55:15.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: cluster 2023-10-09T05:55:12.984566+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: cluster 2023-10-09T05:55:12.984698+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: audit 2023-10-09T05:55:14.722959+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: cluster 2023-10-09T05:55:14.729351+0000 mon.a (mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647] boot 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: cluster 2023-10-09T05:55:14.729379+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: audit 2023-10-09T05:55:14.729742+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: audit 2023-10-09T05:55:14.730081+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: audit 2023-10-09T05:55:15.201890+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[27513]: audit 2023-10-09T05:55:15.206304+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: cluster 2023-10-09T05:55:12.984566+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: cluster 2023-10-09T05:55:12.984698+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: audit 2023-10-09T05:55:14.722959+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: cluster 2023-10-09T05:55:14.729351+0000 mon.a (mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647] boot 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: cluster 2023-10-09T05:55:14.729379+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: audit 2023-10-09T05:55:14.729742+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:15.983 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: audit 2023-10-09T05:55:14.730081+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:55:15.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: audit 2023-10-09T05:55:15.201890+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:15.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:15 smithi125 bash[33919]: audit 2023-10-09T05:55:15.206304+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: cluster 2023-10-09T05:55:12.984566+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: cluster 2023-10-09T05:55:12.984698+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: audit 2023-10-09T05:55:14.722959+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: cluster 2023-10-09T05:55:14.729351+0000 mon.a (mon.0) 329 : cluster [INF] osd.2 [v2:172.21.15.125:6818/3371306647,v1:172.21.15.125:6819/3371306647] boot 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: cluster 2023-10-09T05:55:14.729379+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: audit 2023-10-09T05:55:14.729742+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: audit 2023-10-09T05:55:14.730081+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: audit 2023-10-09T05:55:15.201890+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:15 smithi155 bash[27817]: audit 2023-10-09T05:55:15.206304+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:16.196 INFO:journalctl@ceph.osd.2.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[27513]: audit 2023-10-09T05:55:15.738204+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[27513]: cluster 2023-10-09T05:55:15.738420+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[27513]: audit 2023-10-09T05:55:15.739697+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[27513]: cluster 2023-10-09T05:55:15.880324+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[33919]: audit 2023-10-09T05:55:15.738204+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:55:17.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[33919]: cluster 2023-10-09T05:55:15.738420+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:55:17.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[33919]: audit 2023-10-09T05:55:15.739697+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:55:17.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:16 smithi125 bash[33919]: cluster 2023-10-09T05:55:15.880324+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:17.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:16 smithi155 bash[27817]: audit 2023-10-09T05:55:15.738204+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:55:17.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:16 smithi155 bash[27817]: cluster 2023-10-09T05:55:15.738420+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:55:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:16 smithi155 bash[27817]: audit 2023-10-09T05:55:15.739697+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:55:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:16 smithi155 bash[27817]: cluster 2023-10-09T05:55:15.880324+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:18.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:17 smithi125 bash[27513]: audit 2023-10-09T05:55:16.754383+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:55:18.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:17 smithi125 bash[27513]: cluster 2023-10-09T05:55:16.754465+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:55:18.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:17 smithi125 bash[33919]: audit 2023-10-09T05:55:16.754383+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:55:18.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:17 smithi125 bash[33919]: cluster 2023-10-09T05:55:16.754465+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:55:18.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:17 smithi155 bash[27817]: audit 2023-10-09T05:55:16.754383+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:55:18.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:17 smithi155 bash[27817]: cluster 2023-10-09T05:55:16.754465+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:55:19.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:18 smithi125 bash[33919]: cluster 2023-10-09T05:55:17.753400+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:55:19.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:18 smithi125 bash[33919]: cluster 2023-10-09T05:55:17.880859+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:19.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:18 smithi125 bash[27513]: cluster 2023-10-09T05:55:17.753400+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:55:19.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:18 smithi125 bash[27513]: cluster 2023-10-09T05:55:17.880859+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:19.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:18 smithi155 bash[27817]: cluster 2023-10-09T05:55:17.753400+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:55:19.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:18 smithi155 bash[27817]: cluster 2023-10-09T05:55:17.880859+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:19.523 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:55:19.523 INFO:teuthology.orchestra.run.smithi125.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:55:19.523 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10+0 records in 2023-10-09T05:55:19.523 INFO:teuthology.orchestra.run.smithi125.stderr:10+0 records out 2023-10-09T05:55:19.524 INFO:teuthology.orchestra.run.smithi125.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0152944 s, 686 MB/s 2023-10-09T05:55:19.524 INFO:teuthology.orchestra.run.smithi125.stderr:--> Zapping successful for: 2023-10-09T05:55:20.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi125:vg_nvme/lv_1 2023-10-09T05:55:21.255 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:21 smithi125 bash[27513]: cluster 2023-10-09T05:55:19.881502+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:21.400 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:21 smithi125 bash[33919]: cluster 2023-10-09T05:55:19.881502+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:21.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:21 smithi155 bash[27817]: cluster 2023-10-09T05:55:19.881502+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:23.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:23 smithi125 bash[33919]: cluster 2023-10-09T05:55:21.881966+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:23 smithi125 bash[27513]: cluster 2023-10-09T05:55:21.881966+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:23.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:23 smithi155 bash[27817]: cluster 2023-10-09T05:55:21.881966+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[33919]: cluster 2023-10-09T05:55:23.882482+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[33919]: audit 2023-10-09T05:55:25.190055+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[33919]: audit 2023-10-09T05:55:25.193044+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[33919]: audit 2023-10-09T05:55:25.193873+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[27513]: cluster 2023-10-09T05:55:23.882482+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:25.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[27513]: audit 2023-10-09T05:55:25.190055+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:25.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[27513]: audit 2023-10-09T05:55:25.193044+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:25.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:25 smithi125 bash[27513]: audit 2023-10-09T05:55:25.193873+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:25.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:25 smithi155 bash[27817]: cluster 2023-10-09T05:55:23.882482+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:25 smithi155 bash[27817]: audit 2023-10-09T05:55:25.190055+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:25 smithi155 bash[27817]: audit 2023-10-09T05:55:25.193044+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:25 smithi155 bash[27817]: audit 2023-10-09T05:55:25.193873+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:26.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:26 smithi125 bash[27513]: audit 2023-10-09T05:55:25.187155+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:26.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:26 smithi125 bash[33919]: audit 2023-10-09T05:55:25.187155+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:26.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:26 smithi155 bash[27817]: audit 2023-10-09T05:55:25.187155+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi125:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: cluster 2023-10-09T05:55:25.883002+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.901832+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.903768+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.905050+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.906054+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.913499+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.914994+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.916164+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[27513]: audit 2023-10-09T05:55:26.924722+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: cluster 2023-10-09T05:55:25.883002+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.901832+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.903768+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.905050+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.906054+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.913499+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.914994+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:27.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.916164+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:27.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:27 smithi125 bash[33919]: audit 2023-10-09T05:55:26.924722+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: cluster 2023-10-09T05:55:25.883002+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.901832+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.903768+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.905050+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.906054+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.913499+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.914994+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.916164+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:27 smithi155 bash[27817]: audit 2023-10-09T05:55:26.924722+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:28.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:28 smithi125 bash[33919]: cephadm 2023-10-09T05:55:26.892598+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:55:28.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:28 smithi125 bash[33919]: cephadm 2023-10-09T05:55:26.906895+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 5413M 2023-10-09T05:55:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:28 smithi125 bash[27513]: cephadm 2023-10-09T05:55:26.892598+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:55:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:28 smithi125 bash[27513]: cephadm 2023-10-09T05:55:26.906895+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 5413M 2023-10-09T05:55:28.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:28 smithi155 bash[27817]: cephadm 2023-10-09T05:55:26.892598+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:55:28.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:28 smithi155 bash[27817]: cephadm 2023-10-09T05:55:26.906895+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 5413M 2023-10-09T05:55:29.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:29 smithi125 bash[33919]: cluster 2023-10-09T05:55:27.883519+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:29 smithi125 bash[27513]: cluster 2023-10-09T05:55:27.883519+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:29.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:29 smithi155 bash[27817]: cluster 2023-10-09T05:55:27.883519+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:31.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:31 smithi125 bash[27513]: cluster 2023-10-09T05:55:29.884018+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:31.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:31 smithi125 bash[33919]: cluster 2023-10-09T05:55:29.884018+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:31 smithi155 bash[27817]: cluster 2023-10-09T05:55:29.884018+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:32.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:32 smithi125 bash[27513]: cluster 2023-10-09T05:55:31.884616+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:32.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:32 smithi125 bash[33919]: cluster 2023-10-09T05:55:31.884616+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:32.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:32 smithi155 bash[27817]: cluster 2023-10-09T05:55:31.884616+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:34.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[27513]: audit 2023-10-09T05:55:33.663024+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]: dispatch 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[27513]: audit 2023-10-09T05:55:33.671049+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]': finished 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[27513]: cluster 2023-10-09T05:55:33.671180+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[27513]: audit 2023-10-09T05:55:33.671426+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[33919]: audit 2023-10-09T05:55:33.663024+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]: dispatch 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[33919]: audit 2023-10-09T05:55:33.671049+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]': finished 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[33919]: cluster 2023-10-09T05:55:33.671180+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:55:34.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:33 smithi125 bash[33919]: audit 2023-10-09T05:55:33.671426+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:33 smithi155 bash[27817]: audit 2023-10-09T05:55:33.663024+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]: dispatch 2023-10-09T05:55:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:33 smithi155 bash[27817]: audit 2023-10-09T05:55:33.671049+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.125:0/15634630' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "94e49f30-1edd-4b60-8ac7-1b957d1d2dd9"}]': finished 2023-10-09T05:55:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:33 smithi155 bash[27817]: cluster 2023-10-09T05:55:33.671180+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:55:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:33 smithi155 bash[27817]: audit 2023-10-09T05:55:33.671426+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:35.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:34 smithi125 bash[27513]: cluster 2023-10-09T05:55:33.885231+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:35.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:34 smithi125 bash[27513]: audit 2023-10-09T05:55:34.414937+0000 mon.a (mon.0) 356 : audit [DBG] from='client.? 172.21.15.125:0/2711541376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:35.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:34 smithi125 bash[33919]: cluster 2023-10-09T05:55:33.885231+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:35.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:34 smithi125 bash[33919]: audit 2023-10-09T05:55:34.414937+0000 mon.a (mon.0) 356 : audit [DBG] from='client.? 172.21.15.125:0/2711541376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:35.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:34 smithi155 bash[27817]: cluster 2023-10-09T05:55:33.885231+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:35.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:34 smithi155 bash[27817]: audit 2023-10-09T05:55:34.414937+0000 mon.a (mon.0) 356 : audit [DBG] from='client.? 172.21.15.125:0/2711541376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:37.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:37 smithi125 bash[27513]: cluster 2023-10-09T05:55:35.885783+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:37.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:37 smithi125 bash[33919]: cluster 2023-10-09T05:55:35.885783+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:37.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:37 smithi155 bash[27817]: cluster 2023-10-09T05:55:35.885783+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:39.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:39 smithi125 bash[27513]: cluster 2023-10-09T05:55:37.886271+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:39.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:39 smithi125 bash[33919]: cluster 2023-10-09T05:55:37.886271+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:39.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:39 smithi155 bash[27817]: cluster 2023-10-09T05:55:37.886271+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:41.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:41 smithi125 bash[27513]: cluster 2023-10-09T05:55:39.887048+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:41.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:41 smithi125 bash[33919]: cluster 2023-10-09T05:55:39.887048+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:41.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:41 smithi155 bash[27817]: cluster 2023-10-09T05:55:39.887048+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:43.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:43 smithi125 bash[27513]: cluster 2023-10-09T05:55:41.887590+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:43.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:43 smithi125 bash[33919]: cluster 2023-10-09T05:55:41.887590+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:43.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:43 smithi155 bash[27817]: cluster 2023-10-09T05:55:41.887590+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:45.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:45 smithi125 bash[27513]: cluster 2023-10-09T05:55:43.888217+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:45.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:45 smithi125 bash[33919]: cluster 2023-10-09T05:55:43.888217+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:45.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:45 smithi155 bash[27817]: cluster 2023-10-09T05:55:43.888217+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:46.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:46 smithi125 bash[33919]: audit 2023-10-09T05:55:46.243733+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:55:46.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:46 smithi125 bash[33919]: audit 2023-10-09T05:55:46.245338+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:46.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:46 smithi125 bash[27513]: audit 2023-10-09T05:55:46.243733+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:55:46.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:46 smithi125 bash[27513]: audit 2023-10-09T05:55:46.245338+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:46.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:46 smithi155 bash[27817]: audit 2023-10-09T05:55:46.243733+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:55:46.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:46 smithi155 bash[27817]: audit 2023-10-09T05:55:46.245338+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:47.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:47 smithi125 bash[27513]: cluster 2023-10-09T05:55:45.888622+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:47.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:47 smithi125 bash[27513]: cephadm 2023-10-09T05:55:46.246764+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi125 2023-10-09T05:55:47.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:47 smithi125 bash[33919]: cluster 2023-10-09T05:55:45.888622+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:47.558 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:47 smithi125 bash[33919]: cephadm 2023-10-09T05:55:46.246764+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi125 2023-10-09T05:55:47.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:47 smithi155 bash[27817]: cluster 2023-10-09T05:55:45.888622+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:47 smithi155 bash[27817]: cephadm 2023-10-09T05:55:46.246764+0000 mgr.y (mgr.14146) 149 : cephadm [INF] Deploying daemon osd.3 on smithi125 2023-10-09T05:55:49.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:49 smithi125 bash[27513]: cluster 2023-10-09T05:55:47.889100+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:49.628 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:49 smithi125 bash[33919]: cluster 2023-10-09T05:55:47.889100+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:49 smithi155 bash[27817]: cluster 2023-10-09T05:55:47.889100+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[27513]: cluster 2023-10-09T05:55:49.889462+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[27513]: audit 2023-10-09T05:55:50.665564+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[27513]: audit 2023-10-09T05:55:50.701136+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[33919]: cluster 2023-10-09T05:55:49.889462+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[33919]: audit 2023-10-09T05:55:50.665564+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:51.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:51 smithi125 bash[33919]: audit 2023-10-09T05:55:50.701136+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:51.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:51 smithi155 bash[27817]: cluster 2023-10-09T05:55:49.889462+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:51 smithi155 bash[27817]: audit 2023-10-09T05:55:50.665564+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:51 smithi155 bash[27817]: audit 2023-10-09T05:55:50.701136+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:53.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:53 smithi125 bash[33919]: cluster 2023-10-09T05:55:51.889781+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:53.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:53 smithi125 bash[27513]: cluster 2023-10-09T05:55:51.889781+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:53 smithi155 bash[27817]: cluster 2023-10-09T05:55:51.889781+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:55.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:55 smithi125 bash[27513]: cluster 2023-10-09T05:55:53.890343+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:55.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:55 smithi125 bash[33919]: cluster 2023-10-09T05:55:53.890343+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:55.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:55 smithi155 bash[27817]: cluster 2023-10-09T05:55:53.890343+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:57.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[27513]: cluster 2023-10-09T05:55:55.890742+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[27513]: audit 2023-10-09T05:55:56.058649+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[27513]: audit 2023-10-09T05:55:56.059793+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[27513]: audit 2023-10-09T05:55:56.060801+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[27513]: audit 2023-10-09T05:55:56.067040+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[33919]: cluster 2023-10-09T05:55:55.890742+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[33919]: audit 2023-10-09T05:55:56.058649+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[33919]: audit 2023-10-09T05:55:56.059793+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:57.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[33919]: audit 2023-10-09T05:55:56.060801+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:57.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:57 smithi125 bash[33919]: audit 2023-10-09T05:55:56.067040+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:57.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:57 smithi155 bash[27817]: cluster 2023-10-09T05:55:55.890742+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:57 smithi155 bash[27817]: audit 2023-10-09T05:55:56.058649+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:57 smithi155 bash[27817]: audit 2023-10-09T05:55:56.059793+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:57 smithi155 bash[27817]: audit 2023-10-09T05:55:56.060801+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:57 smithi155 bash[27817]: audit 2023-10-09T05:55:56.067040+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:55:58.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:58 smithi125 bash[27513]: audit 2023-10-09T05:55:57.500882+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:55:58.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:58 smithi125 bash[33919]: audit 2023-10-09T05:55:57.500882+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:55:58.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:58 smithi155 bash[27817]: audit 2023-10-09T05:55:57.500882+0000 mon.a (mon.0) 365 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:55:59.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[27513]: cluster 2023-10-09T05:55:57.891178+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[27513]: audit 2023-10-09T05:55:58.069434+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[27513]: cluster 2023-10-09T05:55:58.069534+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[27513]: audit 2023-10-09T05:55:58.069826+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[27513]: audit 2023-10-09T05:55:58.070374+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[33919]: cluster 2023-10-09T05:55:57.891178+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[33919]: audit 2023-10-09T05:55:58.069434+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[33919]: cluster 2023-10-09T05:55:58.069534+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:55:59.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[33919]: audit 2023-10-09T05:55:58.069826+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:59.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:55:59 smithi125 bash[33919]: audit 2023-10-09T05:55:58.070374+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:59 smithi155 bash[27817]: cluster 2023-10-09T05:55:57.891178+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:55:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:59 smithi155 bash[27817]: audit 2023-10-09T05:55:58.069434+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:55:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:59 smithi155 bash[27817]: cluster 2023-10-09T05:55:58.069534+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:55:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:59 smithi155 bash[27817]: audit 2023-10-09T05:55:58.069826+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:55:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:55:59 smithi155 bash[27817]: audit 2023-10-09T05:55:58.070374+0000 mon.a (mon.0) 369 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T05:55:59.778 INFO:teuthology.orchestra.run.smithi125.stdout:Created osd(s) 3 on host 'smithi125' 2023-10-09T05:56:00.084 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: audit 2023-10-09T05:55:59.072447+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:56:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: audit 2023-10-09T05:55:59.072447+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:56:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: cluster 2023-10-09T05:55:59.072569+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:56:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: audit 2023-10-09T05:55:59.073268+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: audit 2023-10-09T05:55:59.079857+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: audit 2023-10-09T05:55:59.776432+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:00.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[27513]: audit 2023-10-09T05:55:59.794001+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: cluster 2023-10-09T05:55:59.072569+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:56:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: audit 2023-10-09T05:55:59.073268+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: audit 2023-10-09T05:55:59.079857+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: audit 2023-10-09T05:55:59.776432+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:00 smithi125 bash[33919]: audit 2023-10-09T05:55:59.794001+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:00.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: audit 2023-10-09T05:55:59.072447+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]': finished 2023-10-09T05:56:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: cluster 2023-10-09T05:55:59.072569+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:56:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: audit 2023-10-09T05:55:59.073268+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: audit 2023-10-09T05:55:59.079857+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: audit 2023-10-09T05:55:59.776432+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:00 smithi155 bash[27817]: audit 2023-10-09T05:55:59.794001+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:00.532 DEBUG:teuthology.orchestra.run.smithi125:osd.3> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T05:56:00.535 INFO:tasks.cephadm:Deploying osd.4 on smithi155 with /dev/vg_nvme/lv_4... 2023-10-09T05:56:00.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:56:00.646 INFO:journalctl@ceph.osd.3.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:56:01.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: cluster 2023-10-09T05:55:58.454557+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: cluster 2023-10-09T05:55:58.454692+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: cluster 2023-10-09T05:55:59.891481+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: cluster 2023-10-09T05:56:00.080820+0000 mon.a (mon.0) 376 : cluster [INF] osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933] boot 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: cluster 2023-10-09T05:56:00.080863+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[27513]: audit 2023-10-09T05:56:00.081281+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: cluster 2023-10-09T05:55:58.454557+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:01.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: cluster 2023-10-09T05:55:58.454692+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:01.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: cluster 2023-10-09T05:55:59.891481+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:56:01.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: cluster 2023-10-09T05:56:00.080820+0000 mon.a (mon.0) 376 : cluster [INF] osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933] boot 2023-10-09T05:56:01.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: cluster 2023-10-09T05:56:00.080863+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:56:01.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:01 smithi125 bash[33919]: audit 2023-10-09T05:56:00.081281+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:01.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: cluster 2023-10-09T05:55:58.454557+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: cluster 2023-10-09T05:55:58.454692+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: cluster 2023-10-09T05:55:59.891481+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:56:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: cluster 2023-10-09T05:56:00.080820+0000 mon.a (mon.0) 376 : cluster [INF] osd.3 [v2:172.21.15.125:6826/909255933,v1:172.21.15.125:6827/909255933] boot 2023-10-09T05:56:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: cluster 2023-10-09T05:56:00.080863+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:56:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:01 smithi155 bash[27817]: audit 2023-10-09T05:56:00.081281+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:02.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:02 smithi125 bash[33919]: cluster 2023-10-09T05:56:01.267854+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:56:02.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:02 smithi125 bash[27513]: cluster 2023-10-09T05:56:01.267854+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:56:02.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:02 smithi155 bash[27817]: cluster 2023-10-09T05:56:01.267854+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:56:03.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:03 smithi125 bash[27513]: cluster 2023-10-09T05:56:01.891954+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:03.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:03 smithi125 bash[33919]: cluster 2023-10-09T05:56:01.891954+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:03.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:03 smithi155 bash[27817]: cluster 2023-10-09T05:56:01.891954+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0167476 s, 626 MB/s 2023-10-09T05:56:03.925 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T05:56:04.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_4 2023-10-09T05:56:05.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:05 smithi155 bash[27817]: cluster 2023-10-09T05:56:03.892486+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:05.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:05 smithi125 bash[27513]: cluster 2023-10-09T05:56:03.892486+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:05.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:05 smithi125 bash[33919]: cluster 2023-10-09T05:56:03.892486+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:07.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:07 smithi125 bash[27513]: cluster 2023-10-09T05:56:05.893106+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:07.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:07 smithi125 bash[33919]: cluster 2023-10-09T05:56:05.893106+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:07.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:07 smithi155 bash[27817]: cluster 2023-10-09T05:56:05.893106+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:08.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[27513]: audit 2023-10-09T05:56:07.538793+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:08.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[27513]: audit 2023-10-09T05:56:07.543931+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:08.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[27513]: audit 2023-10-09T05:56:07.545235+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:08.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[33919]: audit 2023-10-09T05:56:07.538793+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:08.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[33919]: audit 2023-10-09T05:56:07.543931+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:08.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:08 smithi125 bash[33919]: audit 2023-10-09T05:56:07.545235+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:08.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:08 smithi155 bash[27817]: audit 2023-10-09T05:56:07.538793+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:08 smithi155 bash[27817]: audit 2023-10-09T05:56:07.543931+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:08 smithi155 bash[27817]: audit 2023-10-09T05:56:07.545235+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:07.535412+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: cluster 2023-10-09T05:56:07.893627+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.861955+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.863871+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.865153+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.866341+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.867522+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.872609+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.874713+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.875919+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[27513]: audit 2023-10-09T05:56:08.882589+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:07.535412+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: cluster 2023-10-09T05:56:07.893627+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.861955+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.863871+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.865153+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.866341+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.867522+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.872609+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.874713+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.875919+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:09 smithi125 bash[33919]: audit 2023-10-09T05:56:08.882589+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:07.535412+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: cluster 2023-10-09T05:56:07.893627+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.861955+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.863871+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.865153+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.866341+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.867522+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.872609+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.874713+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.875919+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:09 smithi155 bash[27817]: audit 2023-10-09T05:56:08.882589+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:10.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[27513]: cephadm 2023-10-09T05:56:08.852971+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:56:10.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[27513]: cephadm 2023-10-09T05:56:08.868509+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 4060M 2023-10-09T05:56:10.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[27513]: cluster 2023-10-09T05:56:09.894160+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:10.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[33919]: cephadm 2023-10-09T05:56:08.852971+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:56:10.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[33919]: cephadm 2023-10-09T05:56:08.868509+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 4060M 2023-10-09T05:56:10.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:10 smithi125 bash[33919]: cluster 2023-10-09T05:56:09.894160+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:10.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:10 smithi155 bash[27817]: cephadm 2023-10-09T05:56:08.852971+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi125 2023-10-09T05:56:10.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:10 smithi155 bash[27817]: cephadm 2023-10-09T05:56:08.868509+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 4060M 2023-10-09T05:56:10.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:10 smithi155 bash[27817]: cluster 2023-10-09T05:56:09.894160+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:13.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:12 smithi155 bash[27817]: cluster 2023-10-09T05:56:11.894807+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:13.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:12 smithi125 bash[33919]: cluster 2023-10-09T05:56:11.894807+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:12 smithi125 bash[27513]: cluster 2023-10-09T05:56:11.894807+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:15.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:14 smithi125 bash[27513]: cluster 2023-10-09T05:56:13.895273+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:15.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:14 smithi125 bash[33919]: cluster 2023-10-09T05:56:13.895273+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:15.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:14 smithi155 bash[27817]: cluster 2023-10-09T05:56:13.895273+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:16.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[27513]: audit 2023-10-09T05:56:15.813742+0000 mon.b (mon.1) 4 : audit [INF] from='client.? 172.21.15.155:0/1296065368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[27513]: audit 2023-10-09T05:56:15.814737+0000 mon.a (mon.0) 392 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[27513]: audit 2023-10-09T05:56:15.823144+0000 mon.a (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]': finished 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[27513]: cluster 2023-10-09T05:56:15.823283+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[27513]: audit 2023-10-09T05:56:15.823518+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[33919]: audit 2023-10-09T05:56:15.813742+0000 mon.b (mon.1) 4 : audit [INF] from='client.? 172.21.15.155:0/1296065368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[33919]: audit 2023-10-09T05:56:15.814737+0000 mon.a (mon.0) 392 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[33919]: audit 2023-10-09T05:56:15.823144+0000 mon.a (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]': finished 2023-10-09T05:56:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[33919]: cluster 2023-10-09T05:56:15.823283+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:56:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:16 smithi125 bash[33919]: audit 2023-10-09T05:56:15.823518+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:16 smithi155 bash[27817]: audit 2023-10-09T05:56:15.813742+0000 mon.b (mon.1) 4 : audit [INF] from='client.? 172.21.15.155:0/1296065368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:16 smithi155 bash[27817]: audit 2023-10-09T05:56:15.814737+0000 mon.a (mon.0) 392 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]: dispatch 2023-10-09T05:56:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:16 smithi155 bash[27817]: audit 2023-10-09T05:56:15.823144+0000 mon.a (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd"}]': finished 2023-10-09T05:56:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:16 smithi155 bash[27817]: cluster 2023-10-09T05:56:15.823283+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:56:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:16 smithi155 bash[27817]: audit 2023-10-09T05:56:15.823518+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:17.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:17 smithi125 bash[33919]: cluster 2023-10-09T05:56:15.895799+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:17.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:17 smithi125 bash[33919]: audit 2023-10-09T05:56:16.519980+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.155:0/1689509266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:56:17.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:17 smithi125 bash[27513]: cluster 2023-10-09T05:56:15.895799+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:17.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:17 smithi125 bash[27513]: audit 2023-10-09T05:56:16.519980+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.155:0/1689509266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:56:17.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:17 smithi155 bash[27817]: cluster 2023-10-09T05:56:15.895799+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:17.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:17 smithi155 bash[27817]: audit 2023-10-09T05:56:16.519980+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.155:0/1689509266' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:56:19.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:19 smithi125 bash[27513]: cluster 2023-10-09T05:56:17.896365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:19 smithi125 bash[33919]: cluster 2023-10-09T05:56:17.896365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:19.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:19 smithi155 bash[27817]: cluster 2023-10-09T05:56:17.896365+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:21.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:21 smithi125 bash[27513]: cluster 2023-10-09T05:56:19.896873+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:21 smithi125 bash[33919]: cluster 2023-10-09T05:56:19.896873+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:21 smithi155 bash[27817]: cluster 2023-10-09T05:56:19.896873+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:23.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:23 smithi125 bash[27513]: cluster 2023-10-09T05:56:21.897425+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:23.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:23 smithi125 bash[33919]: cluster 2023-10-09T05:56:21.897425+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:23.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:23 smithi155 bash[27817]: cluster 2023-10-09T05:56:21.897425+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:25.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:25 smithi125 bash[33919]: cluster 2023-10-09T05:56:23.897993+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:25 smithi125 bash[27513]: cluster 2023-10-09T05:56:23.897993+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:25.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:25 smithi155 bash[27817]: cluster 2023-10-09T05:56:23.897993+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:27.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:27 smithi155 bash[27817]: cluster 2023-10-09T05:56:25.898653+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:27.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:27 smithi125 bash[33919]: cluster 2023-10-09T05:56:25.898653+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:27 smithi125 bash[27513]: cluster 2023-10-09T05:56:25.898653+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[27513]: cluster 2023-10-09T05:56:27.899337+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[27513]: audit 2023-10-09T05:56:28.106730+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[27513]: audit 2023-10-09T05:56:28.108811+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[27513]: cephadm 2023-10-09T05:56:28.110521+0000 mgr.y (mgr.14146) 174 : cephadm [INF] Deploying daemon osd.4 on smithi155 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[33919]: cluster 2023-10-09T05:56:27.899337+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[33919]: audit 2023-10-09T05:56:28.106730+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:56:28.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[33919]: audit 2023-10-09T05:56:28.108811+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:28.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:28 smithi125 bash[33919]: cephadm 2023-10-09T05:56:28.110521+0000 mgr.y (mgr.14146) 174 : cephadm [INF] Deploying daemon osd.4 on smithi155 2023-10-09T05:56:28.667 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:28 smithi155 bash[27817]: cluster 2023-10-09T05:56:27.899337+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:28.667 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:28 smithi155 bash[27817]: audit 2023-10-09T05:56:28.106730+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:56:28.667 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:28 smithi155 bash[27817]: audit 2023-10-09T05:56:28.108811+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:28.667 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:28 smithi155 bash[27817]: cephadm 2023-10-09T05:56:28.110521+0000 mgr.y (mgr.14146) 174 : cephadm [INF] Deploying daemon osd.4 on smithi155 2023-10-09T05:56:31.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:31 smithi125 bash[27513]: cluster 2023-10-09T05:56:29.899993+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:31 smithi125 bash[33919]: cluster 2023-10-09T05:56:29.899993+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:31 smithi155 bash[27817]: cluster 2023-10-09T05:56:29.899993+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[27513]: cluster 2023-10-09T05:56:31.900603+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[27513]: audit 2023-10-09T05:56:32.465076+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[27513]: audit 2023-10-09T05:56:32.466129+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[33919]: cluster 2023-10-09T05:56:31.900603+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[33919]: audit 2023-10-09T05:56:32.465076+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:33 smithi125 bash[33919]: audit 2023-10-09T05:56:32.466129+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:33 smithi155 bash[27817]: cluster 2023-10-09T05:56:31.900603+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:33 smithi155 bash[27817]: audit 2023-10-09T05:56:32.465076+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:33 smithi155 bash[27817]: audit 2023-10-09T05:56:32.466129+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:35.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:35 smithi125 bash[27513]: cluster 2023-10-09T05:56:33.901168+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:35.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:35 smithi125 bash[33919]: cluster 2023-10-09T05:56:33.901168+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:35.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:35 smithi155 bash[27817]: cluster 2023-10-09T05:56:33.901168+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:37.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:37 smithi125 bash[27513]: cluster 2023-10-09T05:56:35.901720+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:37.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:37 smithi125 bash[33919]: cluster 2023-10-09T05:56:35.901720+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:37.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:37 smithi155 bash[27817]: cluster 2023-10-09T05:56:35.901720+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[27513]: audit 2023-10-09T05:56:37.532193+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[27513]: audit 2023-10-09T05:56:37.807197+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[27513]: audit 2023-10-09T05:56:37.809079+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[27513]: audit 2023-10-09T05:56:37.817042+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[27513]: cluster 2023-10-09T05:56:37.902261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:38.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[33919]: audit 2023-10-09T05:56:37.532193+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[33919]: audit 2023-10-09T05:56:37.807197+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:38.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[33919]: audit 2023-10-09T05:56:37.809079+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:38.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[33919]: audit 2023-10-09T05:56:37.817042+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:38 smithi125 bash[33919]: cluster 2023-10-09T05:56:37.902261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:38.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:38 smithi155 bash[27817]: audit 2023-10-09T05:56:37.532193+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:38 smithi155 bash[27817]: audit 2023-10-09T05:56:37.807197+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:38 smithi155 bash[27817]: audit 2023-10-09T05:56:37.809079+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:38 smithi155 bash[27817]: audit 2023-10-09T05:56:37.817042+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:38 smithi155 bash[27817]: cluster 2023-10-09T05:56:37.902261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:40.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:39 smithi125 bash[27513]: audit 2023-10-09T05:56:39.014312+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:40.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:39 smithi125 bash[27513]: audit 2023-10-09T05:56:39.014927+0000 mon.a (mon.0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:40.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:39 smithi125 bash[33919]: audit 2023-10-09T05:56:39.014312+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:40.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:39 smithi125 bash[33919]: audit 2023-10-09T05:56:39.014927+0000 mon.a (mon.0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:40.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:39 smithi155 bash[27817]: audit 2023-10-09T05:56:39.014312+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:40.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:39 smithi155 bash[27817]: audit 2023-10-09T05:56:39.014927+0000 mon.a (mon.0) 404 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: audit 2023-10-09T05:56:39.827742+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: cluster 2023-10-09T05:56:39.827831+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: audit 2023-10-09T05:56:39.827995+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: audit 2023-10-09T05:56:39.831299+0000 mon.b (mon.1) 7 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: audit 2023-10-09T05:56:39.831640+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[27513]: cluster 2023-10-09T05:56:39.902798+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: audit 2023-10-09T05:56:39.827742+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: cluster 2023-10-09T05:56:39.827831+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: audit 2023-10-09T05:56:39.827995+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: audit 2023-10-09T05:56:39.831299+0000 mon.b (mon.1) 7 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: audit 2023-10-09T05:56:39.831640+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:40 smithi125 bash[33919]: cluster 2023-10-09T05:56:39.902798+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:41.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: audit 2023-10-09T05:56:39.827742+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:56:41.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: cluster 2023-10-09T05:56:39.827831+0000 mon.a (mon.0) 406 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:56:41.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: audit 2023-10-09T05:56:39.827995+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: audit 2023-10-09T05:56:39.831299+0000 mon.b (mon.1) 7 : audit [INF] from='osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: audit 2023-10-09T05:56:39.831640+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:56:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:40 smithi155 bash[27817]: cluster 2023-10-09T05:56:39.902798+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:56:41.311 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 4 on host 'smithi155' 2023-10-09T05:56:41.981 DEBUG:teuthology.orchestra.run.smithi155:osd.4> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T05:56:41.984 INFO:tasks.cephadm:Deploying osd.5 on smithi155 with /dev/vg_nvme/lv_3... 2023-10-09T05:56:41.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: audit 2023-10-09T05:56:40.833134+0000 mon.a (mon.0) 409 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: cluster 2023-10-09T05:56:40.833315+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: audit 2023-10-09T05:56:40.834109+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: audit 2023-10-09T05:56:40.839595+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: audit 2023-10-09T05:56:41.309931+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: audit 2023-10-09T05:56:41.345909+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:42.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[33919]: cluster 2023-10-09T05:56:41.835558+0000 mon.a (mon.0) 415 : cluster [INF] osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050] boot 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: audit 2023-10-09T05:56:40.833134+0000 mon.a (mon.0) 409 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: cluster 2023-10-09T05:56:40.833315+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: audit 2023-10-09T05:56:40.834109+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: audit 2023-10-09T05:56:40.839595+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: audit 2023-10-09T05:56:41.309931+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: audit 2023-10-09T05:56:41.345909+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:41 smithi125 bash[27513]: cluster 2023-10-09T05:56:41.835558+0000 mon.a (mon.0) 415 : cluster [INF] osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050] boot 2023-10-09T05:56:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: audit 2023-10-09T05:56:40.833134+0000 mon.a (mon.0) 409 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:56:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: cluster 2023-10-09T05:56:40.833315+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: audit 2023-10-09T05:56:40.834109+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: audit 2023-10-09T05:56:40.839595+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: audit 2023-10-09T05:56:41.309931+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: audit 2023-10-09T05:56:41.345909+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:41 smithi155 bash[27817]: cluster 2023-10-09T05:56:41.835558+0000 mon.a (mon.0) 415 : cluster [INF] osd.4 [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050] boot 2023-10-09T05:56:42.221 INFO:journalctl@ceph.osd.4.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:56:43.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[27513]: cluster 2023-10-09T05:56:40.056922+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:43.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[27513]: cluster 2023-10-09T05:56:40.057006+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[27513]: cluster 2023-10-09T05:56:41.835635+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[27513]: audit 2023-10-09T05:56:41.835937+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[27513]: cluster 2023-10-09T05:56:41.903344+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[33919]: cluster 2023-10-09T05:56:40.056922+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[33919]: cluster 2023-10-09T05:56:40.057006+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[33919]: cluster 2023-10-09T05:56:41.835635+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:56:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[33919]: audit 2023-10-09T05:56:41.835937+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:43.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:42 smithi125 bash[33919]: cluster 2023-10-09T05:56:41.903344+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:43.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:42 smithi155 bash[27817]: cluster 2023-10-09T05:56:40.056922+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:56:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:42 smithi155 bash[27817]: cluster 2023-10-09T05:56:40.057006+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:56:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:42 smithi155 bash[27817]: cluster 2023-10-09T05:56:41.835635+0000 mon.a (mon.0) 416 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:56:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:42 smithi155 bash[27817]: audit 2023-10-09T05:56:41.835937+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:42 smithi155 bash[27817]: cluster 2023-10-09T05:56:41.903344+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:44.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:43 smithi125 bash[27513]: cluster 2023-10-09T05:56:42.838529+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:56:44.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:43 smithi125 bash[33919]: cluster 2023-10-09T05:56:42.838529+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:56:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:43 smithi155 bash[27817]: cluster 2023-10-09T05:56:42.838529+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:56:45.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:44 smithi125 bash[27513]: cluster 2023-10-09T05:56:43.848657+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:56:45.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:44 smithi125 bash[27513]: cluster 2023-10-09T05:56:43.903923+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:45.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:44 smithi125 bash[33919]: cluster 2023-10-09T05:56:43.848657+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:56:45.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:44 smithi125 bash[33919]: cluster 2023-10-09T05:56:43.903923+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:45.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:44 smithi155 bash[27817]: cluster 2023-10-09T05:56:43.848657+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:56:45.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:44 smithi155 bash[27817]: cluster 2023-10-09T05:56:43.903923+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0164123 s, 639 MB/s 2023-10-09T05:56:45.403 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T05:56:46.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_3 2023-10-09T05:56:47.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:47 smithi155 bash[27817]: cluster 2023-10-09T05:56:45.904574+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:47.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:47 smithi125 bash[27513]: cluster 2023-10-09T05:56:45.904574+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:47.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:47 smithi125 bash[33919]: cluster 2023-10-09T05:56:45.904574+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:49.563 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:49 smithi155 bash[27817]: cluster 2023-10-09T05:56:47.905265+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:49.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:49 smithi125 bash[33919]: cluster 2023-10-09T05:56:47.905265+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:49 smithi125 bash[27513]: cluster 2023-10-09T05:56:47.905265+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[33919]: cluster 2023-10-09T05:56:49.905956+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[33919]: audit 2023-10-09T05:56:50.813654+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[33919]: audit 2023-10-09T05:56:50.817682+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[33919]: audit 2023-10-09T05:56:50.819005+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[27513]: cluster 2023-10-09T05:56:49.905956+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:51.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[27513]: audit 2023-10-09T05:56:50.813654+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:51.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[27513]: audit 2023-10-09T05:56:50.817682+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:51.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:51 smithi125 bash[27513]: audit 2023-10-09T05:56:50.819005+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:51.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:51 smithi155 bash[27817]: cluster 2023-10-09T05:56:49.905956+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:51 smithi155 bash[27817]: audit 2023-10-09T05:56:50.813654+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:56:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:51 smithi155 bash[27817]: audit 2023-10-09T05:56:50.817682+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:56:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:51 smithi155 bash[27817]: audit 2023-10-09T05:56:50.819005+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:50.806748+0000 mgr.y (mgr.14146) 186 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.112046+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.113759+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.119683+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.121506+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.122879+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:52.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[27513]: audit 2023-10-09T05:56:52.129350+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:50.806748+0000 mgr.y (mgr.14146) 186 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.112046+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.113759+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.119683+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.121506+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.122879+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:52.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:52 smithi125 bash[33919]: audit 2023-10-09T05:56:52.129350+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:50.806748+0000 mgr.y (mgr.14146) 186 : audit [DBG] from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.112046+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.113759+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.119683+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.121506+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.122879+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:52 smithi155 bash[27817]: audit 2023-10-09T05:56:52.129350+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[27513]: cluster 2023-10-09T05:56:51.906514+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[27513]: cephadm 2023-10-09T05:56:52.102877+0000 mgr.y (mgr.14146) 188 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[27513]: cephadm 2023-10-09T05:56:52.114920+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 17265M 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[33919]: cluster 2023-10-09T05:56:51.906514+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[33919]: cephadm 2023-10-09T05:56:52.102877+0000 mgr.y (mgr.14146) 188 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:56:53.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:53 smithi125 bash[33919]: cephadm 2023-10-09T05:56:52.114920+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 17265M 2023-10-09T05:56:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:53 smithi155 bash[27817]: cluster 2023-10-09T05:56:51.906514+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:53 smithi155 bash[27817]: cephadm 2023-10-09T05:56:52.102877+0000 mgr.y (mgr.14146) 188 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:56:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:53 smithi155 bash[27817]: cephadm 2023-10-09T05:56:52.114920+0000 mgr.y (mgr.14146) 189 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 17265M 2023-10-09T05:56:54.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:54 smithi125 bash[33919]: cluster 2023-10-09T05:56:53.907019+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:54.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:54 smithi125 bash[27513]: cluster 2023-10-09T05:56:53.907019+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:54.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:54 smithi155 bash[27817]: cluster 2023-10-09T05:56:53.907019+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:57.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:57 smithi125 bash[27513]: cluster 2023-10-09T05:56:55.907554+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:57.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:57 smithi125 bash[33919]: cluster 2023-10-09T05:56:55.907554+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:57 smithi155 bash[27817]: cluster 2023-10-09T05:56:55.907554+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: cluster 2023-10-09T05:56:57.908075+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: audit 2023-10-09T05:56:59.009167+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.155:0/6009837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: audit 2023-10-09T05:56:59.010141+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: audit 2023-10-09T05:56:59.019261+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]': finished 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: cluster 2023-10-09T05:56:59.019397+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[33919]: audit 2023-10-09T05:56:59.019580+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:56:59.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: cluster 2023-10-09T05:56:57.908075+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: audit 2023-10-09T05:56:59.009167+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.155:0/6009837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: audit 2023-10-09T05:56:59.010141+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: audit 2023-10-09T05:56:59.019261+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]': finished 2023-10-09T05:56:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: cluster 2023-10-09T05:56:59.019397+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:56:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:56:59 smithi125 bash[27513]: audit 2023-10-09T05:56:59.019580+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:56:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: cluster 2023-10-09T05:56:57.908075+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:56:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: audit 2023-10-09T05:56:59.009167+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.155:0/6009837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: audit 2023-10-09T05:56:59.010141+0000 mon.a (mon.0) 429 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]: dispatch 2023-10-09T05:56:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: audit 2023-10-09T05:56:59.019261+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79c3a7df-d176-4f67-8de0-1bb46949eeba"}]': finished 2023-10-09T05:56:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: cluster 2023-10-09T05:56:59.019397+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-09T05:56:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:56:59 smithi155 bash[27817]: audit 2023-10-09T05:56:59.019580+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:00.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:00 smithi125 bash[27513]: audit 2023-10-09T05:56:59.715672+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.155:0/3779914736' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:00.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:00 smithi125 bash[33919]: audit 2023-10-09T05:56:59.715672+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.155:0/3779914736' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:00.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:00 smithi155 bash[27817]: audit 2023-10-09T05:56:59.715672+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.155:0/3779914736' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:01 smithi125 bash[27513]: cluster 2023-10-09T05:56:59.908605+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:01.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:01 smithi125 bash[33919]: cluster 2023-10-09T05:56:59.908605+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:01 smithi155 bash[27817]: cluster 2023-10-09T05:56:59.908605+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:03.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:03 smithi125 bash[27513]: cluster 2023-10-09T05:57:01.909186+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:03 smithi125 bash[33919]: cluster 2023-10-09T05:57:01.909186+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:03.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:03 smithi155 bash[27817]: cluster 2023-10-09T05:57:01.909186+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:05.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:05 smithi125 bash[27513]: cluster 2023-10-09T05:57:03.909751+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:05 smithi125 bash[33919]: cluster 2023-10-09T05:57:03.909751+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:05 smithi155 bash[27817]: cluster 2023-10-09T05:57:03.909751+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:06.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:06 smithi125 bash[33919]: cluster 2023-10-09T05:57:05.910287+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:06.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:06 smithi125 bash[27513]: cluster 2023-10-09T05:57:05.910287+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:06.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:06 smithi155 bash[27817]: cluster 2023-10-09T05:57:05.910287+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:09.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:08 smithi125 bash[33919]: cluster 2023-10-09T05:57:07.910960+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:09.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:08 smithi125 bash[27513]: cluster 2023-10-09T05:57:07.910960+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:09.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:09 smithi155 bash[27817]: cluster 2023-10-09T05:57:07.910960+0000 mgr.y (mgr.14146) 197 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:11.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:11 smithi155 bash[27817]: cluster 2023-10-09T05:57:09.911638+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:11.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:11 smithi125 bash[27513]: cluster 2023-10-09T05:57:09.911638+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:11.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:11 smithi125 bash[33919]: cluster 2023-10-09T05:57:09.911638+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:12.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:12 smithi155 bash[27817]: audit 2023-10-09T05:57:11.338744+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:57:12.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:12 smithi155 bash[27817]: audit 2023-10-09T05:57:11.340627+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:12 smithi125 bash[27513]: audit 2023-10-09T05:57:11.338744+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:57:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:12 smithi125 bash[27513]: audit 2023-10-09T05:57:11.340627+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:12 smithi125 bash[33919]: audit 2023-10-09T05:57:11.338744+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:57:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:12 smithi125 bash[33919]: audit 2023-10-09T05:57:11.340627+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:13.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:13 smithi125 bash[33919]: cephadm 2023-10-09T05:57:11.341934+0000 mgr.y (mgr.14146) 199 : cephadm [INF] Deploying daemon osd.5 on smithi155 2023-10-09T05:57:13.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:13 smithi125 bash[33919]: cluster 2023-10-09T05:57:11.912248+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:13.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:13 smithi125 bash[27513]: cephadm 2023-10-09T05:57:11.341934+0000 mgr.y (mgr.14146) 199 : cephadm [INF] Deploying daemon osd.5 on smithi155 2023-10-09T05:57:13.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:13 smithi125 bash[27513]: cluster 2023-10-09T05:57:11.912248+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:13.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:13 smithi155 bash[27817]: cephadm 2023-10-09T05:57:11.341934+0000 mgr.y (mgr.14146) 199 : cephadm [INF] Deploying daemon osd.5 on smithi155 2023-10-09T05:57:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:13 smithi155 bash[27817]: cluster 2023-10-09T05:57:11.912248+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:15.307 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:15 smithi155 bash[27817]: cluster 2023-10-09T05:57:13.912793+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:15.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:15 smithi125 bash[33919]: cluster 2023-10-09T05:57:13.912793+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:15.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:15 smithi125 bash[27513]: cluster 2023-10-09T05:57:13.912793+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:16.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:16 smithi155 bash[27817]: audit 2023-10-09T05:57:15.494808+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:16 smithi155 bash[27817]: audit 2023-10-09T05:57:15.511306+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:16 smithi155 bash[27817]: cluster 2023-10-09T05:57:15.913314+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[27513]: audit 2023-10-09T05:57:15.494808+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[27513]: audit 2023-10-09T05:57:15.511306+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[27513]: cluster 2023-10-09T05:57:15.913314+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[33919]: audit 2023-10-09T05:57:15.494808+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[33919]: audit 2023-10-09T05:57:15.511306+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:16.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:16 smithi125 bash[33919]: cluster 2023-10-09T05:57:15.913314+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:19.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:18 smithi125 bash[27513]: cluster 2023-10-09T05:57:17.913867+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:19.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:18 smithi125 bash[33919]: cluster 2023-10-09T05:57:17.913867+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:18 smithi155 bash[27817]: cluster 2023-10-09T05:57:17.913867+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:21.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:21 smithi155 bash[27817]: cluster 2023-10-09T05:57:19.914395+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:21.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:21 smithi155 bash[27817]: audit 2023-10-09T05:57:20.603831+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:21.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:21 smithi155 bash[27817]: audit 2023-10-09T05:57:20.605086+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:21.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:21 smithi155 bash[27817]: audit 2023-10-09T05:57:20.606452+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:21.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:21 smithi155 bash[27817]: audit 2023-10-09T05:57:20.613106+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[27513]: cluster 2023-10-09T05:57:19.914395+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[27513]: audit 2023-10-09T05:57:20.603831+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[27513]: audit 2023-10-09T05:57:20.605086+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[27513]: audit 2023-10-09T05:57:20.606452+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[27513]: audit 2023-10-09T05:57:20.613106+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[33919]: cluster 2023-10-09T05:57:19.914395+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[33919]: audit 2023-10-09T05:57:20.603831+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[33919]: audit 2023-10-09T05:57:20.605086+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[33919]: audit 2023-10-09T05:57:20.606452+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:21 smithi125 bash[33919]: audit 2023-10-09T05:57:20.613106+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:22.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:22 smithi155 bash[27817]: audit 2023-10-09T05:57:22.144820+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:22.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:22 smithi155 bash[27817]: audit 2023-10-09T05:57:22.145435+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:22.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:22 smithi125 bash[27513]: audit 2023-10-09T05:57:22.144820+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:22.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:22 smithi125 bash[27513]: audit 2023-10-09T05:57:22.145435+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:22.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:22 smithi125 bash[33919]: audit 2023-10-09T05:57:22.144820+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:22.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:22 smithi125 bash[33919]: audit 2023-10-09T05:57:22.145435+0000 mon.a (mon.0) 441 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: cluster 2023-10-09T05:57:21.914990+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: audit 2023-10-09T05:57:22.299029+0000 mon.a (mon.0) 442 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: cluster 2023-10-09T05:57:22.299138+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: audit 2023-10-09T05:57:22.299302+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: audit 2023-10-09T05:57:22.300023+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:23.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[27513]: audit 2023-10-09T05:57:22.300353+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: cluster 2023-10-09T05:57:21.914990+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: audit 2023-10-09T05:57:22.299029+0000 mon.a (mon.0) 442 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: cluster 2023-10-09T05:57:22.299138+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: audit 2023-10-09T05:57:22.299302+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: audit 2023-10-09T05:57:22.300023+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:23 smithi125 bash[33919]: audit 2023-10-09T05:57:22.300353+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:23.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: cluster 2023-10-09T05:57:21.914990+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:23.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: audit 2023-10-09T05:57:22.299029+0000 mon.a (mon.0) 442 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:57:23.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: cluster 2023-10-09T05:57:22.299138+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-09T05:57:23.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: audit 2023-10-09T05:57:22.299302+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:23.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: audit 2023-10-09T05:57:22.300023+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:23.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:23 smithi155 bash[27817]: audit 2023-10-09T05:57:22.300353+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:57:24.205 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 5 on host 'smithi155' 2023-10-09T05:57:24.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: audit 2023-10-09T05:57:23.302340+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: cluster 2023-10-09T05:57:23.302664+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: audit 2023-10-09T05:57:23.303614+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: audit 2023-10-09T05:57:23.309602+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: audit 2023-10-09T05:57:24.203472+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[27513]: audit 2023-10-09T05:57:24.240735+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: audit 2023-10-09T05:57:23.302340+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: cluster 2023-10-09T05:57:23.302664+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:57:24.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: audit 2023-10-09T05:57:23.303614+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: audit 2023-10-09T05:57:23.309602+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: audit 2023-10-09T05:57:24.203472+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:24 smithi125 bash[33919]: audit 2023-10-09T05:57:24.240735+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: audit 2023-10-09T05:57:23.302340+0000 mon.a (mon.0) 446 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: cluster 2023-10-09T05:57:23.302664+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: audit 2023-10-09T05:57:23.303614+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: audit 2023-10-09T05:57:23.309602+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: audit 2023-10-09T05:57:24.203472+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:24 smithi155 bash[27817]: audit 2023-10-09T05:57:24.240735+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:24.908 DEBUG:teuthology.orchestra.run.smithi155:osd.5> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T05:57:24.910 INFO:tasks.cephadm:Deploying osd.6 on smithi155 with /dev/vg_nvme/lv_2... 2023-10-09T05:57:24.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:57:25.098 INFO:journalctl@ceph.osd.5.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:23.142235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:23.142334+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:23.915536+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:24.304989+0000 mon.a (mon.0) 452 : cluster [INF] osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641] boot 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:24.305110+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:57:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: audit 2023-10-09T05:57:24.305412+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[27513]: cluster 2023-10-09T05:57:25.288856+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:23.142235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:23.142334+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:23.915536+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:24.304989+0000 mon.a (mon.0) 452 : cluster [INF] osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641] boot 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:24.305110+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: audit 2023-10-09T05:57:24.305412+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:25 smithi125 bash[33919]: cluster 2023-10-09T05:57:25.288856+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:23.142235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:23.142334+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:23.915536+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:24.304989+0000 mon.a (mon.0) 452 : cluster [INF] osd.5 [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641] boot 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:24.305110+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: audit 2023-10-09T05:57:24.305412+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:57:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:25 smithi155 bash[27817]: cluster 2023-10-09T05:57:25.288856+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-09T05:57:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:27 smithi125 bash[27513]: cluster 2023-10-09T05:57:25.916079+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:27 smithi125 bash[27513]: cluster 2023-10-09T05:57:26.299566+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:57:27.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:27 smithi125 bash[33919]: cluster 2023-10-09T05:57:25.916079+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:27.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:27 smithi125 bash[33919]: cluster 2023-10-09T05:57:26.299566+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:57:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:27 smithi155 bash[27817]: cluster 2023-10-09T05:57:25.916079+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:27 smithi155 bash[27817]: cluster 2023-10-09T05:57:26.299566+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0175688 s, 597 MB/s 2023-10-09T05:57:28.450 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T05:57:29.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_2 2023-10-09T05:57:29.425 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:29 smithi155 bash[27817]: cluster 2023-10-09T05:57:27.916628+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:29 smithi125 bash[27513]: cluster 2023-10-09T05:57:27.916628+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:29.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:29 smithi125 bash[33919]: cluster 2023-10-09T05:57:27.916628+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:31 smithi125 bash[27513]: cluster 2023-10-09T05:57:29.917142+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:31 smithi125 bash[33919]: cluster 2023-10-09T05:57:29.917142+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:31 smithi155 bash[27817]: cluster 2023-10-09T05:57:29.917142+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[27513]: cluster 2023-10-09T05:57:31.917679+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[27513]: audit 2023-10-09T05:57:32.884606+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:57:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[27513]: audit 2023-10-09T05:57:32.888224+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:57:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[27513]: audit 2023-10-09T05:57:32.889492+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:33.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[33919]: cluster 2023-10-09T05:57:31.917679+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:33.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[33919]: audit 2023-10-09T05:57:32.884606+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:57:33.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[33919]: audit 2023-10-09T05:57:32.888224+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:57:33.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:33 smithi125 bash[33919]: audit 2023-10-09T05:57:32.889492+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:33 smithi155 bash[27817]: cluster 2023-10-09T05:57:31.917679+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:33 smithi155 bash[27817]: audit 2023-10-09T05:57:32.884606+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:57:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:33 smithi155 bash[27817]: audit 2023-10-09T05:57:32.888224+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:57:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:33 smithi155 bash[27817]: audit 2023-10-09T05:57:32.889492+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:34 smithi155 bash[27817]: audit 2023-10-09T05:57:32.877738+0000 mgr.y (mgr.14146) 211 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:34.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:34 smithi125 bash[27513]: audit 2023-10-09T05:57:32.877738+0000 mgr.y (mgr.14146) 211 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:34.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:34 smithi125 bash[33919]: audit 2023-10-09T05:57:32.877738+0000 mgr.y (mgr.14146) 211 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:35.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:35 smithi125 bash[27513]: cluster 2023-10-09T05:57:33.918256+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:35.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:35 smithi125 bash[33919]: cluster 2023-10-09T05:57:33.918256+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:35.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:35 smithi155 bash[27817]: cluster 2023-10-09T05:57:33.918256+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:37.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: cephadm 2023-10-09T05:57:35.748019+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:57:37.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.756996+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.758892+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.760364+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: cephadm 2023-10-09T05:57:35.761328+0000 mgr.y (mgr.14146) 214 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 8632M 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.766983+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.768468+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.769591+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: audit 2023-10-09T05:57:35.776689+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[27513]: cluster 2023-10-09T05:57:35.918753+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: cephadm 2023-10-09T05:57:35.748019+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.756996+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.758892+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.760364+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: cephadm 2023-10-09T05:57:35.761328+0000 mgr.y (mgr.14146) 214 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 8632M 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.766983+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.768468+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.769591+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: audit 2023-10-09T05:57:35.776689+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:36 smithi125 bash[33919]: cluster 2023-10-09T05:57:35.918753+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: cephadm 2023-10-09T05:57:35.748019+0000 mgr.y (mgr.14146) 213 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:57:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.756996+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.758892+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.760364+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:57:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: cephadm 2023-10-09T05:57:35.761328+0000 mgr.y (mgr.14146) 214 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 8632M 2023-10-09T05:57:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.766983+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.768468+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.769591+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: audit 2023-10-09T05:57:35.776689+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:36 smithi155 bash[27817]: cluster 2023-10-09T05:57:35.918753+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:39.309 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:38 smithi155 bash[27817]: cluster 2023-10-09T05:57:37.919311+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:39.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:38 smithi125 bash[27513]: cluster 2023-10-09T05:57:37.919311+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:39.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:38 smithi125 bash[33919]: cluster 2023-10-09T05:57:37.919311+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:41.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:41 smithi125 bash[27513]: cluster 2023-10-09T05:57:39.919883+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:41.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:41 smithi125 bash[33919]: cluster 2023-10-09T05:57:39.919883+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:41.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:41 smithi155 bash[27817]: cluster 2023-10-09T05:57:39.919883+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: audit 2023-10-09T05:57:41.495260+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.155:0/2683470919' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: audit 2023-10-09T05:57:41.496160+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: audit 2023-10-09T05:57:41.504221+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]': finished 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: cluster 2023-10-09T05:57:41.504347+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: audit 2023-10-09T05:57:41.504625+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:57:42.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[27513]: audit 2023-10-09T05:57:42.214378+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.155:0/2938796773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: audit 2023-10-09T05:57:41.495260+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.155:0/2683470919' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: audit 2023-10-09T05:57:41.496160+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: audit 2023-10-09T05:57:41.504221+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]': finished 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: cluster 2023-10-09T05:57:41.504347+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: audit 2023-10-09T05:57:41.504625+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:57:42.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:42 smithi125 bash[33919]: audit 2023-10-09T05:57:42.214378+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.155:0/2938796773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:42.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: audit 2023-10-09T05:57:41.495260+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.155:0/2683470919' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: audit 2023-10-09T05:57:41.496160+0000 mon.a (mon.0) 467 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]: dispatch 2023-10-09T05:57:42.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: audit 2023-10-09T05:57:41.504221+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65cb2125-86cc-4e3c-9bb6-3926ff3db18e"}]': finished 2023-10-09T05:57:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: cluster 2023-10-09T05:57:41.504347+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-09T05:57:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: audit 2023-10-09T05:57:41.504625+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:57:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:42 smithi155 bash[27817]: audit 2023-10-09T05:57:42.214378+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.155:0/2938796773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:57:43.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:43 smithi125 bash[27513]: cluster 2023-10-09T05:57:41.920442+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:43.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:43 smithi125 bash[33919]: cluster 2023-10-09T05:57:41.920442+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:43.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:43 smithi155 bash[27817]: cluster 2023-10-09T05:57:41.920442+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:45.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:45 smithi125 bash[27513]: cluster 2023-10-09T05:57:43.921006+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:45.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:45 smithi125 bash[33919]: cluster 2023-10-09T05:57:43.921006+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:45.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:45 smithi155 bash[27817]: cluster 2023-10-09T05:57:43.921006+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:47.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:47 smithi125 bash[33919]: cluster 2023-10-09T05:57:45.921502+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:47 smithi125 bash[27513]: cluster 2023-10-09T05:57:45.921502+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:47.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:47 smithi155 bash[27817]: cluster 2023-10-09T05:57:45.921502+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:49.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:49 smithi125 bash[33919]: cluster 2023-10-09T05:57:47.922101+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:49 smithi125 bash[27513]: cluster 2023-10-09T05:57:47.922101+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:49.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:49 smithi155 bash[27817]: cluster 2023-10-09T05:57:47.922101+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:51 smithi155 bash[27817]: cluster 2023-10-09T05:57:49.922705+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:51.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:51 smithi125 bash[33919]: cluster 2023-10-09T05:57:49.922705+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:51.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:51 smithi125 bash[27513]: cluster 2023-10-09T05:57:49.922705+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:53.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:53 smithi125 bash[33919]: cluster 2023-10-09T05:57:51.923209+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:53.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:53 smithi125 bash[27513]: cluster 2023-10-09T05:57:51.923209+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:53 smithi155 bash[27817]: cluster 2023-10-09T05:57:51.923209+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:54.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:54 smithi125 bash[27513]: audit 2023-10-09T05:57:54.015379+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:57:54.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:54 smithi125 bash[27513]: audit 2023-10-09T05:57:54.017152+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:54.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:54 smithi125 bash[33919]: audit 2023-10-09T05:57:54.015379+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:57:54.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:54 smithi125 bash[33919]: audit 2023-10-09T05:57:54.017152+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:54.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:54 smithi155 bash[27817]: audit 2023-10-09T05:57:54.015379+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:57:54.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:54 smithi155 bash[27817]: audit 2023-10-09T05:57:54.017152+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:55 smithi125 bash[27513]: cluster 2023-10-09T05:57:53.923758+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:55.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:55 smithi125 bash[27513]: cephadm 2023-10-09T05:57:54.018669+0000 mgr.y (mgr.14146) 225 : cephadm [INF] Deploying daemon osd.6 on smithi155 2023-10-09T05:57:55.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:55 smithi125 bash[33919]: cluster 2023-10-09T05:57:53.923758+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:55.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:55 smithi125 bash[33919]: cephadm 2023-10-09T05:57:54.018669+0000 mgr.y (mgr.14146) 225 : cephadm [INF] Deploying daemon osd.6 on smithi155 2023-10-09T05:57:55.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:55 smithi155 bash[27817]: cluster 2023-10-09T05:57:53.923758+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:55.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:55 smithi155 bash[27817]: cephadm 2023-10-09T05:57:54.018669+0000 mgr.y (mgr.14146) 225 : cephadm [INF] Deploying daemon osd.6 on smithi155 2023-10-09T05:57:57.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:57 smithi125 bash[27513]: cluster 2023-10-09T05:57:55.924307+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:57.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:57 smithi125 bash[33919]: cluster 2023-10-09T05:57:55.924307+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:57.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:57 smithi155 bash[27817]: cluster 2023-10-09T05:57:55.924307+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:58.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:58 smithi125 bash[33919]: cluster 2023-10-09T05:57:57.924898+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:58.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:58 smithi125 bash[27513]: cluster 2023-10-09T05:57:57.924898+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:58.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:58 smithi155 bash[27817]: cluster 2023-10-09T05:57:57.924898+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:57:59.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:59 smithi155 bash[27817]: audit 2023-10-09T05:57:58.390520+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:57:59 smithi155 bash[27817]: audit 2023-10-09T05:57:58.405988+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:59.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:59 smithi125 bash[27513]: audit 2023-10-09T05:57:58.390520+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:59.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:57:59 smithi125 bash[27513]: audit 2023-10-09T05:57:58.405988+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:59.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:59 smithi125 bash[33919]: audit 2023-10-09T05:57:58.390520+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:57:59.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:57:59 smithi125 bash[33919]: audit 2023-10-09T05:57:58.405988+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:01 smithi125 bash[27513]: cluster 2023-10-09T05:57:59.925467+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:01.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:01 smithi125 bash[33919]: cluster 2023-10-09T05:57:59.925467+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:01 smithi155 bash[27817]: cluster 2023-10-09T05:57:59.925467+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:03.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:03 smithi125 bash[27513]: cluster 2023-10-09T05:58:01.926015+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:03.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:03 smithi125 bash[33919]: cluster 2023-10-09T05:58:01.926015+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:03.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:03 smithi155 bash[27817]: cluster 2023-10-09T05:58:01.926015+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[27513]: audit 2023-10-09T05:58:03.533729+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[27513]: audit 2023-10-09T05:58:03.821350+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[27513]: audit 2023-10-09T05:58:03.823116+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[27513]: audit 2023-10-09T05:58:03.831002+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[27513]: cluster 2023-10-09T05:58:03.926571+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[33919]: audit 2023-10-09T05:58:03.533729+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[33919]: audit 2023-10-09T05:58:03.821350+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[33919]: audit 2023-10-09T05:58:03.823116+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[33919]: audit 2023-10-09T05:58:03.831002+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:04 smithi125 bash[33919]: cluster 2023-10-09T05:58:03.926571+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:04 smithi155 bash[27817]: audit 2023-10-09T05:58:03.533729+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:04 smithi155 bash[27817]: audit 2023-10-09T05:58:03.821350+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:04 smithi155 bash[27817]: audit 2023-10-09T05:58:03.823116+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:04 smithi155 bash[27817]: audit 2023-10-09T05:58:03.831002+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:04 smithi155 bash[27817]: cluster 2023-10-09T05:58:03.926571+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:06.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:05 smithi125 bash[27513]: audit 2023-10-09T05:58:05.127487+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:05 smithi125 bash[27513]: audit 2023-10-09T05:58:05.128113+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:05 smithi125 bash[33919]: audit 2023-10-09T05:58:05.127487+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:05 smithi125 bash[33919]: audit 2023-10-09T05:58:05.128113+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:05 smithi155 bash[27817]: audit 2023-10-09T05:58:05.127487+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:05 smithi155 bash[27817]: audit 2023-10-09T05:58:05.128113+0000 mon.a (mon.0) 479 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: audit 2023-10-09T05:58:05.843322+0000 mon.a (mon.0) 480 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: cluster 2023-10-09T05:58:05.843410+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: audit 2023-10-09T05:58:05.843614+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: audit 2023-10-09T05:58:05.846367+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: audit 2023-10-09T05:58:05.846746+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:06 smithi155 bash[27817]: cluster 2023-10-09T05:58:05.927095+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: audit 2023-10-09T05:58:05.843322+0000 mon.a (mon.0) 480 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: cluster 2023-10-09T05:58:05.843410+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: audit 2023-10-09T05:58:05.843614+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: audit 2023-10-09T05:58:05.846367+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: audit 2023-10-09T05:58:05.846746+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:07.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[27513]: cluster 2023-10-09T05:58:05.927095+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: audit 2023-10-09T05:58:05.843322+0000 mon.a (mon.0) 480 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: cluster 2023-10-09T05:58:05.843410+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: audit 2023-10-09T05:58:05.843614+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: audit 2023-10-09T05:58:05.846367+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: audit 2023-10-09T05:58:05.846746+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:07.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:06 smithi125 bash[33919]: cluster 2023-10-09T05:58:05.927095+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:07.163 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 6 on host 'smithi155' 2023-10-09T05:58:07.875 DEBUG:teuthology.orchestra.run.smithi155:osd.6> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T05:58:07.878 INFO:tasks.cephadm:Deploying osd.7 on smithi155 with /dev/vg_nvme/lv_1... 2023-10-09T05:58:07.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:58:08.070 INFO:journalctl@ceph.osd.6.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:58:08.115 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: audit 2023-10-09T05:58:06.847235+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:08.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: cluster 2023-10-09T05:58:06.847359+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:58:08.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: audit 2023-10-09T05:58:06.848215+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: audit 2023-10-09T05:58:06.853897+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: audit 2023-10-09T05:58:07.161343+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:08.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:07 smithi155 bash[27817]: audit 2023-10-09T05:58:07.162069+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: audit 2023-10-09T05:58:06.847235+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: cluster 2023-10-09T05:58:06.847359+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: audit 2023-10-09T05:58:06.848215+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: audit 2023-10-09T05:58:06.853897+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: audit 2023-10-09T05:58:07.161343+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:08.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[27513]: audit 2023-10-09T05:58:07.162069+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: audit 2023-10-09T05:58:06.847235+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: cluster 2023-10-09T05:58:06.847359+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: audit 2023-10-09T05:58:06.848215+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: audit 2023-10-09T05:58:06.853897+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: audit 2023-10-09T05:58:07.161343+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:08.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:07 smithi125 bash[33919]: audit 2023-10-09T05:58:07.162069+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: cluster 2023-10-09T05:58:06.112263+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: cluster 2023-10-09T05:58:06.112344+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: cluster 2023-10-09T05:58:07.857762+0000 mon.a (mon.0) 490 : cluster [INF] osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062] boot 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: cluster 2023-10-09T05:58:07.857820+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: audit 2023-10-09T05:58:07.858402+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:09.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[27513]: cluster 2023-10-09T05:58:07.927580+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: cluster 2023-10-09T05:58:06.112263+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: cluster 2023-10-09T05:58:06.112344+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: cluster 2023-10-09T05:58:07.857762+0000 mon.a (mon.0) 490 : cluster [INF] osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062] boot 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: cluster 2023-10-09T05:58:07.857820+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: audit 2023-10-09T05:58:07.858402+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:09.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:08 smithi125 bash[33919]: cluster 2023-10-09T05:58:07.927580+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:09.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: cluster 2023-10-09T05:58:06.112263+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: cluster 2023-10-09T05:58:06.112344+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: cluster 2023-10-09T05:58:07.857762+0000 mon.a (mon.0) 490 : cluster [INF] osd.6 [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062] boot 2023-10-09T05:58:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: cluster 2023-10-09T05:58:07.857820+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-09T05:58:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: audit 2023-10-09T05:58:07.858402+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:58:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:08 smithi155 bash[27817]: cluster 2023-10-09T05:58:07.927580+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:58:10.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:09 smithi125 bash[27513]: cluster 2023-10-09T05:58:08.859772+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:58:10.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:09 smithi125 bash[33919]: cluster 2023-10-09T05:58:08.859772+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:58:10.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:09 smithi155 bash[27817]: cluster 2023-10-09T05:58:08.859772+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-09T05:58:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:10 smithi125 bash[27513]: cluster 2023-10-09T05:58:09.871347+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:58:11.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:10 smithi125 bash[27513]: cluster 2023-10-09T05:58:09.928185+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v221: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:10 smithi125 bash[33919]: cluster 2023-10-09T05:58:09.871347+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:58:11.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:10 smithi125 bash[33919]: cluster 2023-10-09T05:58:09.928185+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v221: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:10 smithi155 bash[27817]: cluster 2023-10-09T05:58:09.871347+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-09T05:58:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:10 smithi155 bash[27817]: cluster 2023-10-09T05:58:09.928185+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v221: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:11.335 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:58:11.336 INFO:teuthology.orchestra.run.smithi155.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:58:11.336 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10+0 records in 2023-10-09T05:58:11.336 INFO:teuthology.orchestra.run.smithi155.stderr:10+0 records out 2023-10-09T05:58:11.336 INFO:teuthology.orchestra.run.smithi155.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0189439 s, 554 MB/s 2023-10-09T05:58:11.336 INFO:teuthology.orchestra.run.smithi155.stderr:--> Zapping successful for: 2023-10-09T05:58:12.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi155:vg_nvme/lv_1 2023-10-09T05:58:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:11 smithi155 bash[27817]: cluster 2023-10-09T05:58:10.873217+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:12.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:11 smithi125 bash[33919]: cluster 2023-10-09T05:58:10.873217+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:12.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:11 smithi125 bash[27513]: cluster 2023-10-09T05:58:10.873217+0000 mon.a (mon.0) 495 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:58:13.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:12 smithi155 bash[27817]: cluster 2023-10-09T05:58:11.928810+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v222: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:13.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:12 smithi125 bash[27513]: cluster 2023-10-09T05:58:11.928810+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v222: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:13.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:12 smithi125 bash[33919]: cluster 2023-10-09T05:58:11.928810+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v222: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:14.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:14 smithi125 bash[27513]: cluster 2023-10-09T05:58:13.980477+0000 mon.a (mon.0) 496 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:14.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:14 smithi125 bash[27513]: cluster 2023-10-09T05:58:13.980550+0000 mon.a (mon.0) 497 : cluster [INF] Cluster is now healthy 2023-10-09T05:58:14.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:14 smithi125 bash[33919]: cluster 2023-10-09T05:58:13.980477+0000 mon.a (mon.0) 496 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:14.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:14 smithi125 bash[33919]: cluster 2023-10-09T05:58:13.980550+0000 mon.a (mon.0) 497 : cluster [INF] Cluster is now healthy 2023-10-09T05:58:14.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:14 smithi155 bash[27817]: cluster 2023-10-09T05:58:13.980477+0000 mon.a (mon.0) 496 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:58:14.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:14 smithi155 bash[27817]: cluster 2023-10-09T05:58:13.980550+0000 mon.a (mon.0) 497 : cluster [INF] Cluster is now healthy 2023-10-09T05:58:15.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:15 smithi125 bash[27513]: cluster 2023-10-09T05:58:13.929340+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:15.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:15 smithi125 bash[33919]: cluster 2023-10-09T05:58:13.929340+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:15.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:15 smithi155 bash[27817]: cluster 2023-10-09T05:58:13.929340+0000 mgr.y (mgr.14146) 235 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[27513]: audit 2023-10-09T05:58:15.846294+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:58:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[27513]: audit 2023-10-09T05:58:15.849818+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:58:16.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[27513]: audit 2023-10-09T05:58:15.851002+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[33919]: audit 2023-10-09T05:58:15.846294+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:58:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[33919]: audit 2023-10-09T05:58:15.849818+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:58:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:16 smithi125 bash[33919]: audit 2023-10-09T05:58:15.851002+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:16 smithi155 bash[27817]: audit 2023-10-09T05:58:15.846294+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:58:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:16 smithi155 bash[27817]: audit 2023-10-09T05:58:15.849818+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:58:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:16 smithi155 bash[27817]: audit 2023-10-09T05:58:15.851002+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:17.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:17 smithi125 bash[27513]: audit 2023-10-09T05:58:15.839368+0000 mgr.y (mgr.14146) 236 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:17.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:17 smithi125 bash[27513]: cluster 2023-10-09T05:58:15.929959+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:17.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:17 smithi125 bash[33919]: audit 2023-10-09T05:58:15.839368+0000 mgr.y (mgr.14146) 236 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:17.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:17 smithi125 bash[33919]: cluster 2023-10-09T05:58:15.929959+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:17 smithi155 bash[27817]: audit 2023-10-09T05:58:15.839368+0000 mgr.y (mgr.14146) 236 : audit [DBG] from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi155:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:17 smithi155 bash[27817]: cluster 2023-10-09T05:58:15.929959+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: cluster 2023-10-09T05:58:17.930553+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.754654+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.756256+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.757553+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.758806+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.764560+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.766198+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.767453+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[33919]: audit 2023-10-09T05:58:18.773484+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: cluster 2023-10-09T05:58:17.930553+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.754654+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.756256+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.757553+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.758806+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.764560+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.766198+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.767453+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:19 smithi125 bash[27513]: audit 2023-10-09T05:58:18.773484+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: cluster 2023-10-09T05:58:17.930553+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.754654+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.756256+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.757553+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.758806+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.764560+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.766198+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.767453+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:19 smithi155 bash[27817]: audit 2023-10-09T05:58:18.773484+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:20 smithi125 bash[27513]: cephadm 2023-10-09T05:58:18.745044+0000 mgr.y (mgr.14146) 239 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:58:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:20 smithi125 bash[27513]: cephadm 2023-10-09T05:58:18.759754+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 5755M 2023-10-09T05:58:20.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:20 smithi125 bash[33919]: cephadm 2023-10-09T05:58:18.745044+0000 mgr.y (mgr.14146) 239 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:58:20.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:20 smithi125 bash[33919]: cephadm 2023-10-09T05:58:18.759754+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 5755M 2023-10-09T05:58:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:20 smithi155 bash[27817]: cephadm 2023-10-09T05:58:18.745044+0000 mgr.y (mgr.14146) 239 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:58:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:20 smithi155 bash[27817]: cephadm 2023-10-09T05:58:18.759754+0000 mgr.y (mgr.14146) 240 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 5755M 2023-10-09T05:58:21.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:21 smithi125 bash[27513]: cluster 2023-10-09T05:58:19.931052+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:21.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:21 smithi125 bash[33919]: cluster 2023-10-09T05:58:19.931052+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:21 smithi155 bash[27817]: cluster 2023-10-09T05:58:19.931052+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:22.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:22 smithi125 bash[33919]: cluster 2023-10-09T05:58:21.931612+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:22.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:22 smithi125 bash[27513]: cluster 2023-10-09T05:58:21.931612+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:22.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:22 smithi155 bash[27817]: cluster 2023-10-09T05:58:21.931612+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:25.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: cluster 2023-10-09T05:58:23.932208+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:25.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: audit 2023-10-09T05:58:24.565082+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.155:0/4010935334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: audit 2023-10-09T05:58:24.565686+0000 mon.a (mon.0) 509 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: audit 2023-10-09T05:58:24.575713+0000 mon.a (mon.0) 510 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]': finished 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: cluster 2023-10-09T05:58:24.575823+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[33919]: audit 2023-10-09T05:58:24.576085+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: cluster 2023-10-09T05:58:23.932208+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: audit 2023-10-09T05:58:24.565082+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.155:0/4010935334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: audit 2023-10-09T05:58:24.565686+0000 mon.a (mon.0) 509 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: audit 2023-10-09T05:58:24.575713+0000 mon.a (mon.0) 510 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]': finished 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: cluster 2023-10-09T05:58:24.575823+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:58:25.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:24 smithi125 bash[27513]: audit 2023-10-09T05:58:24.576085+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:25.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: cluster 2023-10-09T05:58:23.932208+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: audit 2023-10-09T05:58:24.565082+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.155:0/4010935334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: audit 2023-10-09T05:58:24.565686+0000 mon.a (mon.0) 509 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]: dispatch 2023-10-09T05:58:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: audit 2023-10-09T05:58:24.575713+0000 mon.a (mon.0) 510 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd9d3a50-9683-4779-942e-d41abad90f08"}]': finished 2023-10-09T05:58:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: cluster 2023-10-09T05:58:24.575823+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-09T05:58:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:24 smithi155 bash[27817]: audit 2023-10-09T05:58:24.576085+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:26.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:25 smithi125 bash[27513]: audit 2023-10-09T05:58:25.284656+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.155:0/3607902928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:58:26.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:25 smithi125 bash[33919]: audit 2023-10-09T05:58:25.284656+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.155:0/3607902928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:58:26.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:25 smithi155 bash[27817]: audit 2023-10-09T05:58:25.284656+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.155:0/3607902928' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:58:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:27 smithi125 bash[27513]: cluster 2023-10-09T05:58:25.932807+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:27.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:27 smithi125 bash[33919]: cluster 2023-10-09T05:58:25.932807+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:27 smithi155 bash[27817]: cluster 2023-10-09T05:58:25.932807+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:29 smithi125 bash[27513]: cluster 2023-10-09T05:58:27.933339+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:29.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:29 smithi125 bash[33919]: cluster 2023-10-09T05:58:27.933339+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:29.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:29 smithi155 bash[27817]: cluster 2023-10-09T05:58:27.933339+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:31 smithi125 bash[27513]: cluster 2023-10-09T05:58:29.933915+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:31.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:31 smithi125 bash[33919]: cluster 2023-10-09T05:58:29.933915+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:31.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:31 smithi155 bash[27817]: cluster 2023-10-09T05:58:29.933915+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:33 smithi125 bash[27513]: cluster 2023-10-09T05:58:31.934521+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:33 smithi125 bash[33919]: cluster 2023-10-09T05:58:31.934521+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:33 smithi155 bash[27817]: cluster 2023-10-09T05:58:31.934521+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:34.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:34 smithi155 bash[27817]: cluster 2023-10-09T05:58:33.935097+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:34.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:34 smithi125 bash[27513]: cluster 2023-10-09T05:58:33.935097+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:34.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:34 smithi125 bash[33919]: cluster 2023-10-09T05:58:33.935097+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:37.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[33919]: cluster 2023-10-09T05:58:35.935678+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:37.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[33919]: audit 2023-10-09T05:58:37.103574+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:58:37.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[33919]: audit 2023-10-09T05:58:37.105407+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:37.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[27513]: cluster 2023-10-09T05:58:35.935678+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:37.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[27513]: audit 2023-10-09T05:58:37.103574+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:58:37.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:37 smithi125 bash[27513]: audit 2023-10-09T05:58:37.105407+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:37.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:37 smithi155 bash[27817]: cluster 2023-10-09T05:58:35.935678+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:37.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:37 smithi155 bash[27817]: audit 2023-10-09T05:58:37.103574+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:58:37.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:37 smithi155 bash[27817]: audit 2023-10-09T05:58:37.105407+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:38.563 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:38 smithi155 bash[27817]: cephadm 2023-10-09T05:58:37.106821+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Deploying daemon osd.7 on smithi155 2023-10-09T05:58:38.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:38 smithi125 bash[27513]: cephadm 2023-10-09T05:58:37.106821+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Deploying daemon osd.7 on smithi155 2023-10-09T05:58:38.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:38 smithi125 bash[33919]: cephadm 2023-10-09T05:58:37.106821+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Deploying daemon osd.7 on smithi155 2023-10-09T05:58:39.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:39 smithi155 bash[27817]: cluster 2023-10-09T05:58:37.936254+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:39.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:39 smithi125 bash[33919]: cluster 2023-10-09T05:58:37.936254+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:39.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:39 smithi125 bash[27513]: cluster 2023-10-09T05:58:37.936254+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:41.312 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:41 smithi155 bash[27817]: cluster 2023-10-09T05:58:39.936864+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:41 smithi125 bash[27513]: cluster 2023-10-09T05:58:39.936864+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:41.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:41 smithi125 bash[33919]: cluster 2023-10-09T05:58:39.936864+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:42 smithi155 bash[27817]: audit 2023-10-09T05:58:41.547110+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:42 smithi155 bash[27817]: audit 2023-10-09T05:58:41.554759+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:42.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:42 smithi155 bash[27817]: cluster 2023-10-09T05:58:41.937437+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:42.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[27513]: audit 2023-10-09T05:58:41.547110+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:42.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[27513]: audit 2023-10-09T05:58:41.554759+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:42.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[27513]: cluster 2023-10-09T05:58:41.937437+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:42.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[33919]: audit 2023-10-09T05:58:41.547110+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:42.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[33919]: audit 2023-10-09T05:58:41.554759+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:42.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:42 smithi125 bash[33919]: cluster 2023-10-09T05:58:41.937437+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:45.289 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:44 smithi155 bash[27817]: cluster 2023-10-09T05:58:43.938025+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:45.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:44 smithi125 bash[27513]: cluster 2023-10-09T05:58:43.938025+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:45.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:44 smithi125 bash[33919]: cluster 2023-10-09T05:58:43.938025+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:47.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:47 smithi125 bash[33919]: cluster 2023-10-09T05:58:45.938637+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:47 smithi125 bash[27513]: cluster 2023-10-09T05:58:45.938637+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:47.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:47 smithi155 bash[27817]: cluster 2023-10-09T05:58:45.938637+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:49.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:47.770852+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:47.772260+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:47.773650+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:47.780434+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: cluster 2023-10-09T05:58:47.939141+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:48.262256+0000 mon.c (mon.2) 12 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:49.128 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:48 smithi155 bash[27817]: audit 2023-10-09T05:58:48.263015+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:47.770852+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:47.772260+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:47.773650+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:47.780434+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: cluster 2023-10-09T05:58:47.939141+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:49.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:48.262256+0000 mon.c (mon.2) 12 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[27513]: audit 2023-10-09T05:58:48.263015+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:47.770852+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:47.772260+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:47.773650+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:47.780434+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: cluster 2023-10-09T05:58:47.939141+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:48.262256+0000 mon.c (mon.2) 12 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:49.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:48 smithi125 bash[33919]: audit 2023-10-09T05:58:48.263015+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:58:50.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:49 smithi155 bash[27817]: audit 2023-10-09T05:58:48.784865+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:58:50.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:49 smithi155 bash[27817]: cluster 2023-10-09T05:58:48.784937+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:58:50.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:49 smithi155 bash[27817]: audit 2023-10-09T05:58:48.785145+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:50.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:49 smithi155 bash[27817]: audit 2023-10-09T05:58:48.786064+0000 mon.c (mon.2) 13 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:50.086 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:49 smithi155 bash[27817]: audit 2023-10-09T05:58:48.786556+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:50.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[27513]: audit 2023-10-09T05:58:48.784865+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:58:50.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[27513]: cluster 2023-10-09T05:58:48.784937+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:58:50.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[27513]: audit 2023-10-09T05:58:48.785145+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:50.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[27513]: audit 2023-10-09T05:58:48.786064+0000 mon.c (mon.2) 13 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[27513]: audit 2023-10-09T05:58:48.786556+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[33919]: audit 2023-10-09T05:58:48.784865+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[33919]: cluster 2023-10-09T05:58:48.784937+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[33919]: audit 2023-10-09T05:58:48.785145+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[33919]: audit 2023-10-09T05:58:48.786064+0000 mon.c (mon.2) 13 : audit [INF] from='osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:49 smithi125 bash[33919]: audit 2023-10-09T05:58:48.786556+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]: dispatch 2023-10-09T05:58:51.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: audit 2023-10-09T05:58:49.789844+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: cluster 2023-10-09T05:58:49.790035+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: audit 2023-10-09T05:58:49.791001+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: audit 2023-10-09T05:58:49.798261+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: cluster 2023-10-09T05:58:49.939667+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: cluster 2023-10-09T05:58:50.315035+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035] boot 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: cluster 2023-10-09T05:58:50.315106+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[27513]: audit 2023-10-09T05:58:50.315729+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: audit 2023-10-09T05:58:49.789844+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:51.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: cluster 2023-10-09T05:58:49.790035+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: audit 2023-10-09T05:58:49.791001+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: audit 2023-10-09T05:58:49.798261+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: cluster 2023-10-09T05:58:49.939667+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: cluster 2023-10-09T05:58:50.315035+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035] boot 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: cluster 2023-10-09T05:58:50.315106+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:58:51.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:50 smithi125 bash[33919]: audit 2023-10-09T05:58:50.315729+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: audit 2023-10-09T05:58:49.789844+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi155", "root=default"]}]': finished 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: cluster 2023-10-09T05:58:49.790035+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: audit 2023-10-09T05:58:49.791001+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: audit 2023-10-09T05:58:49.798261+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: cluster 2023-10-09T05:58:49.939667+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: cluster 2023-10-09T05:58:50.315035+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035] boot 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: cluster 2023-10-09T05:58:50.315106+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-09T05:58:51.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:50 smithi155 bash[27817]: audit 2023-10-09T05:58:50.315729+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:58:51.413 INFO:teuthology.orchestra.run.smithi155.stdout:Created osd(s) 7 on host 'smithi155' 2023-10-09T05:58:52.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[27513]: cluster 2023-10-09T05:58:49.214652+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:52.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[27513]: cluster 2023-10-09T05:58:49.214742+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:52.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[27513]: cluster 2023-10-09T05:58:51.323667+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:58:52.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[27513]: audit 2023-10-09T05:58:51.396692+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[27513]: audit 2023-10-09T05:58:51.408752+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[33919]: cluster 2023-10-09T05:58:49.214652+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[33919]: cluster 2023-10-09T05:58:49.214742+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[33919]: cluster 2023-10-09T05:58:51.323667+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[33919]: audit 2023-10-09T05:58:51.396692+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:51 smithi125 bash[33919]: audit 2023-10-09T05:58:51.408752+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:51 smithi155 bash[27817]: cluster 2023-10-09T05:58:49.214652+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-09T05:58:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:51 smithi155 bash[27817]: cluster 2023-10-09T05:58:49.214742+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-09T05:58:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:51 smithi155 bash[27817]: cluster 2023-10-09T05:58:51.323667+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-09T05:58:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:51 smithi155 bash[27817]: audit 2023-10-09T05:58:51.396692+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:58:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:51 smithi155 bash[27817]: audit 2023-10-09T05:58:51.408752+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:52.145 DEBUG:teuthology.orchestra.run.smithi155:osd.7> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T05:58:52.148 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T05:58:52.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:58:52.367 INFO:journalctl@ceph.osd.7.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:58:53.628 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:53 smithi125 bash[27513]: cluster 2023-10-09T05:58:51.940318+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:53.628 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:53 smithi125 bash[27513]: cluster 2023-10-09T05:58:52.327770+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:58:53.628 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:53 smithi125 bash[33919]: cluster 2023-10-09T05:58:51.940318+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:53.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:53 smithi125 bash[33919]: cluster 2023-10-09T05:58:52.327770+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:58:53.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:53 smithi155 bash[27817]: cluster 2023-10-09T05:58:51.940318+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:53 smithi155 bash[27817]: cluster 2023-10-09T05:58:52.327770+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-09T05:58:55.348 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T05:58:55.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:55 smithi125 bash[33919]: cluster 2023-10-09T05:58:53.940952+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:55.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:55 smithi125 bash[27513]: cluster 2023-10-09T05:58:53.940952+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:55 smithi155 bash[27817]: cluster 2023-10-09T05:58:53.940952+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.065 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1696831130,"num_in_osds":8,"osd_in_since":1696831104,"num_remapped_pgs":0} 2023-10-09T05:58:56.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:58:56.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:56 smithi125 bash[27513]: audit 2023-10-09T05:58:55.346806+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.125:0/2537108649' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:58:56.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:56 smithi125 bash[27513]: cluster 2023-10-09T05:58:55.941366+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:56 smithi125 bash[33919]: audit 2023-10-09T05:58:55.346806+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.125:0/2537108649' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:58:56.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:56 smithi125 bash[33919]: cluster 2023-10-09T05:58:55.941366+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:56 smithi155 bash[27817]: audit 2023-10-09T05:58:55.346806+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.125:0/2537108649' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:58:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:56 smithi155 bash[27817]: cluster 2023-10-09T05:58:55.941366+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:59.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:58:58 smithi155 bash[27817]: cluster 2023-10-09T05:58:57.941883+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:59.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:58:58 smithi125 bash[27513]: cluster 2023-10-09T05:58:57.941883+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:59.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:58:58 smithi125 bash[33919]: cluster 2023-10-09T05:58:57.941883+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:00.830 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: cluster 2023-10-09T05:58:59.942510+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.602389+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.604238+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.605635+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.607026+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.608294+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.616400+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.618057+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.619259+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[27513]: audit 2023-10-09T05:59:00.628118+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: cluster 2023-10-09T05:58:59.942510+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.602389+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.604238+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.605635+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.607026+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.608294+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.616400+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.618057+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.619259+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:01.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:01 smithi125 bash[33919]: audit 2023-10-09T05:59:00.628118+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: cluster 2023-10-09T05:58:59.942510+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.602389+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.604238+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.605635+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.607026+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.608294+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:59:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.616400+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.618057+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.619259+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:01 smithi155 bash[27817]: audit 2023-10-09T05:59:00.628118+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:02.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[33919]: cephadm 2023-10-09T05:59:00.589612+0000 mgr.y (mgr.14146) 263 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:59:02.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[33919]: cephadm 2023-10-09T05:59:00.609264+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 4316M 2023-10-09T05:59:02.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[33919]: cluster 2023-10-09T05:59:01.943060+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:02.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[27513]: cephadm 2023-10-09T05:59:00.589612+0000 mgr.y (mgr.14146) 263 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:59:02.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[27513]: cephadm 2023-10-09T05:59:00.609264+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 4316M 2023-10-09T05:59:02.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:02 smithi125 bash[27513]: cluster 2023-10-09T05:59:01.943060+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:02.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:02 smithi155 bash[27817]: cephadm 2023-10-09T05:59:00.589612+0000 mgr.y (mgr.14146) 263 : cephadm [INF] Detected new or changed devices on smithi155 2023-10-09T05:59:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:02 smithi155 bash[27817]: cephadm 2023-10-09T05:59:00.609264+0000 mgr.y (mgr.14146) 264 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 4316M 2023-10-09T05:59:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:02 smithi155 bash[27817]: cluster 2023-10-09T05:59:01.943060+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:03.776 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T05:59:03.776 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":50,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T05:58:52.319219+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:59:04.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:03 smithi125 bash[27513]: audit 2023-10-09T05:59:03.773448+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.125:0/1868311176' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:04.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:03 smithi125 bash[33919]: audit 2023-10-09T05:59:03.773448+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.125:0/1868311176' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:03 smithi155 bash[27817]: audit 2023-10-09T05:59:03.773448+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.125:0/1868311176' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:04.542 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-09T05:55:15.199759+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-09T05:59:04.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-09T05:59:05.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:04 smithi125 bash[27513]: cluster 2023-10-09T05:59:03.943677+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:05.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:04 smithi125 bash[33919]: cluster 2023-10-09T05:59:03.943677+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:05.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:04 smithi155 bash[27817]: cluster 2023-10-09T05:59:03.943677+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:07.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:07 smithi125 bash[27513]: cluster 2023-10-09T05:59:05.944283+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:07.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:07 smithi125 bash[33919]: cluster 2023-10-09T05:59:05.944283+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:07.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:07 smithi155 bash[27817]: cluster 2023-10-09T05:59:05.944283+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:09.303 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:09 smithi125 bash[27513]: cluster 2023-10-09T05:59:07.944953+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:09.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:09 smithi125 bash[33919]: cluster 2023-10-09T05:59:07.944953+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:09 smithi155 bash[27817]: cluster 2023-10-09T05:59:07.944953+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:11.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:11 smithi125 bash[27513]: cluster 2023-10-09T05:59:09.945558+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:11.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:11 smithi125 bash[33919]: cluster 2023-10-09T05:59:09.945558+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:11.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:11 smithi155 bash[27817]: cluster 2023-10-09T05:59:09.945558+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:12.232 INFO:teuthology.orchestra.run.smithi125.stdout:pg_num: 1 2023-10-09T05:59:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:12 smithi125 bash[27513]: audit 2023-10-09T05:59:12.230643+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.125:0/1106340203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:59:12.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:12 smithi125 bash[33919]: audit 2023-10-09T05:59:12.230643+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.125:0/1106340203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:59:12.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:12 smithi155 bash[27817]: audit 2023-10-09T05:59:12.230643+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.125:0/1106340203' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-09T05:59:12.951 INFO:tasks.cephadm:Adding prometheus.a on smithi155 2023-10-09T05:59:12.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi155=a' 2023-10-09T05:59:13.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:13 smithi125 bash[27513]: cluster 2023-10-09T05:59:11.946084+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:13.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:13 smithi125 bash[33919]: cluster 2023-10-09T05:59:11.946084+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:13.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:13 smithi155 bash[27817]: cluster 2023-10-09T05:59:11.946084+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:14.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:14 smithi155 bash[27817]: cluster 2023-10-09T05:59:13.946655+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:14.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:14 smithi125 bash[27513]: cluster 2023-10-09T05:59:13.946655+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:14.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:14 smithi125 bash[33919]: cluster 2023-10-09T05:59:13.946655+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:16.202 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled prometheus update... 2023-10-09T05:59:16.927 DEBUG:teuthology.orchestra.run.smithi155:prometheus.a> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T05:59:16.930 INFO:tasks.cephadm:Adding node-exporter.a on smithi125 2023-10-09T05:59:16.930 INFO:tasks.cephadm:Adding node-exporter.b on smithi155 2023-10-09T05:59:16.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi125=a;smithi155=b' 2023-10-09T05:59:16.973 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: cluster 2023-10-09T05:59:15.947165+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.189991+0000 mgr.y (mgr.14146) 273 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: cephadm 2023-10-09T05:59:16.193384+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.200456+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.236545+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:17.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.529371+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.531262+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.539616+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[27817]: audit 2023-10-09T05:59:16.546572+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: cluster 2023-10-09T05:59:15.947165+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.189991+0000 mgr.y (mgr.14146) 273 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:17.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: cephadm 2023-10-09T05:59:16.193384+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.200456+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.236545+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.529371+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.531262+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.539616+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27513]: audit 2023-10-09T05:59:16.546572+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: cluster 2023-10-09T05:59:15.947165+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.189991+0000 mgr.y (mgr.14146) 273 : audit [DBG] from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: cephadm 2023-10-09T05:59:16.193384+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Saving service prometheus spec with placement smithi155=a;count:1 2023-10-09T05:59:17.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.200456+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.236545+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:17.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.529371+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:17.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.531262+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:17.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.539616+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' 2023-10-09T05:59:17.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[33919]: audit 2023-10-09T05:59:16.546572+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:59:17.859 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27779]: ignoring --setuser ceph since I am not root 2023-10-09T05:59:17.860 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27779]: ignoring --setgroup ceph since I am not root 2023-10-09T05:59:17.860 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27779]: debug 2023-10-09T05:59:17.592+0000 7f03a9b13700 1 -- 172.21.15.125:0/1555556780 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5574ee07e4e0 con 0x5574ee172800 2023-10-09T05:59:17.860 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27779]: debug 2023-10-09T05:59:17.748+0000 7f03b9cdb680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:59:17.861 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[28688]: ignoring --setuser ceph since I am not root 2023-10-09T05:59:17.861 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[28688]: ignoring --setgroup ceph since I am not root 2023-10-09T05:59:17.861 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[28688]: debug 2023-10-09T05:59:17.593+0000 7f50c8df0700 1 -- 172.21.15.155:0/4089669768 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55674ec944e0 con 0x55674ed89000 2023-10-09T05:59:17.861 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[28688]: debug 2023-10-09T05:59:17.749+0000 7f50d8fb8680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:59:17.861 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:17 smithi155 bash[28688]: debug 2023-10-09T05:59:17.857+0000 7f50d8fb8680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:59:18.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:17 smithi125 bash[27779]: debug 2023-10-09T05:59:17.852+0000 7f03b9cdb680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:59:18.545 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:18 smithi155 bash[28688]: debug 2023-10-09T05:59:18.329+0000 7f50d8fb8680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:59:18.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:18 smithi125 bash[33919]: audit 2023-10-09T05:59:17.550228+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:59:18.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:18 smithi125 bash[33919]: cluster 2023-10-09T05:59:17.550315+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T05:59:18.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:18 smithi125 bash[27779]: debug 2023-10-09T05:59:18.328+0000 7f03b9cdb680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:59:18.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:18 smithi125 bash[27513]: audit 2023-10-09T05:59:17.550228+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:59:18.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:18 smithi125 bash[27513]: cluster 2023-10-09T05:59:17.550315+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T05:59:18.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:18 smithi155 bash[27817]: audit 2023-10-09T05:59:17.550228+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14146 172.21.15.125:0/2490675181' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:59:18.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:18 smithi155 bash[27817]: cluster 2023-10-09T05:59:17.550315+0000 mon.a (mon.0) 555 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-09T05:59:19.619 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:19 smithi155 bash[28688]: debug 2023-10-09T05:59:19.261+0000 7f50d8fb8680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:59:19.619 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:19 smithi155 bash[28688]: debug 2023-10-09T05:59:19.361+0000 7f50d8fb8680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:59:19.627 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:19 smithi125 bash[27779]: debug 2023-10-09T05:59:19.268+0000 7f03b9cdb680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:59:19.627 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:19 smithi125 bash[27779]: debug 2023-10-09T05:59:19.368+0000 7f03b9cdb680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:59:19.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:19 smithi125 bash[27779]: debug 2023-10-09T05:59:19.620+0000 7f03b9cdb680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:59:19.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:19 smithi155 bash[28688]: debug 2023-10-09T05:59:19.617+0000 7f50d8fb8680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:59:21.031 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:20 smithi155 bash[28688]: debug 2023-10-09T05:59:20.761+0000 7f50d8fb8680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:59:21.032 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:20 smithi155 bash[28688]: debug 2023-10-09T05:59:20.929+0000 7f50d8fb8680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:59:21.036 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:20 smithi125 bash[27779]: debug 2023-10-09T05:59:20.764+0000 7f03b9cdb680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:59:21.036 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:20 smithi125 bash[27779]: debug 2023-10-09T05:59:20.928+0000 7f03b9cdb680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:59:21.367 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:21 smithi155 bash[28688]: debug 2023-10-09T05:59:21.029+0000 7f50d8fb8680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:59:21.367 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:21 smithi155 bash[28688]: debug 2023-10-09T05:59:21.141+0000 7f50d8fb8680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:59:21.375 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:21 smithi125 bash[27779]: debug 2023-10-09T05:59:21.032+0000 7f03b9cdb680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:59:21.376 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:21 smithi125 bash[27779]: debug 2023-10-09T05:59:21.144+0000 7f03b9cdb680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:59:21.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:21 smithi125 bash[27779]: debug 2023-10-09T05:59:21.372+0000 7f03b9cdb680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:59:21.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:21 smithi155 bash[28688]: debug 2023-10-09T05:59:21.365+0000 7f50d8fb8680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:59:22.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:21 smithi125 bash[27779]: debug 2023-10-09T05:59:21.740+0000 7f03b9cdb680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:59:22.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:21 smithi125 bash[27779]: debug 2023-10-09T05:59:21.860+0000 7f03b9cdb680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:59:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:21 smithi155 bash[28688]: debug 2023-10-09T05:59:21.737+0000 7f50d8fb8680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:59:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:21 smithi155 bash[28688]: debug 2023-10-09T05:59:21.857+0000 7f50d8fb8680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:59:23.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:22 smithi125 bash[27779]: debug 2023-10-09T05:59:22.728+0000 7f03b9cdb680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:59:23.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:22 smithi125 bash[27779]: debug 2023-10-09T05:59:22.832+0000 7f03b9cdb680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:59:23.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:22 smithi125 bash[27779]: debug 2023-10-09T05:59:22.952+0000 7f03b9cdb680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:59:23.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:22 smithi155 bash[28688]: debug 2023-10-09T05:59:22.716+0000 7f50d8fb8680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:59:23.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:22 smithi155 bash[28688]: debug 2023-10-09T05:59:22.820+0000 7f50d8fb8680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:59:23.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:22 smithi155 bash[28688]: debug 2023-10-09T05:59:22.936+0000 7f50d8fb8680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:59:23.455 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:23 smithi155 bash[28688]: debug 2023-10-09T05:59:23.156+0000 7f50d8fb8680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:59:23.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:23 smithi125 bash[27779]: debug 2023-10-09T05:59:23.176+0000 7f03b9cdb680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:59:23.793 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:23 smithi155 bash[28688]: debug 2023-10-09T05:59:23.452+0000 7f50d8fb8680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:59:23.793 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:23 smithi155 bash[28688]: debug 2023-10-09T05:59:23.616+0000 7f50d8fb8680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:59:23.816 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:23 smithi125 bash[27779]: debug 2023-10-09T05:59:23.468+0000 7f03b9cdb680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:59:23.816 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:23 smithi125 bash[27779]: debug 2023-10-09T05:59:23.632+0000 7f03b9cdb680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:59:24.119 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:23 smithi155 bash[28688]: debug 2023-10-09T05:59:23.788+0000 7f50d8fb8680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:59:24.119 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: debug 2023-10-09T05:59:24.008+0000 7f50d8fb8680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:59:24.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:23 smithi125 bash[27779]: debug 2023-10-09T05:59:23.812+0000 7f03b9cdb680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:59:24.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: debug 2023-10-09T05:59:24.028+0000 7f03b9cdb680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:59:24.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: debug 2023-10-09T05:59:24.116+0000 7f50d8fb8680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:59:24.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: [09/Oct/2023:05:59:24] ENGINE Bus STARTING 2023-10-09T05:59:24.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: CherryPy Checker: 2023-10-09T05:59:24.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: The Application mounted at '' has an empty config. 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: [09/Oct/2023:05:59:24] ENGINE Serving on http://:::9283 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[28688]: [09/Oct/2023:05:59:24] ENGINE Bus STARTED 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.140836+0000 mon.a (mon.0) 556 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.141005+0000 mon.a (mon.0) 557 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.144754+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.147460+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:59:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.150703+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.151926+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.160414+0000 mon.a (mon.0) 558 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.160952+0000 mon.a (mon.0) 559 : cluster [INF] Activating manager daemon y 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.171739+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.172142+0000 mon.a (mon.0) 561 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0113494s), standbys: x 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.173584+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.173801+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.173998+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.175155+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.175385+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.175622+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:59:24.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.176001+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.176319+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.176653+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.176955+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.177204+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.177399+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.177665+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.177938+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.178156+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:59:24.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:24 smithi155 bash[27817]: audit 2023-10-09T05:59:24.178740+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:59:24.410 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.140836+0000 mon.a (mon.0) 556 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.141005+0000 mon.a (mon.0) 557 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.144754+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.147460+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.150703+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.151926+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.160414+0000 mon.a (mon.0) 558 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.160952+0000 mon.a (mon.0) 559 : cluster [INF] Activating manager daemon y 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.171739+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.172142+0000 mon.a (mon.0) 561 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0113494s), standbys: x 2023-10-09T05:59:24.411 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.173584+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.173801+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.173998+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.175155+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.175385+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.175622+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.176001+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.176319+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.176653+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.176955+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.177204+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.177399+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:59:24.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.177665+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.177938+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.178156+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[33919]: audit 2023-10-09T05:59:24.178740+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.140836+0000 mon.a (mon.0) 556 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.141005+0000 mon.a (mon.0) 557 : cluster [DBG] Standby manager daemon x started 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.144754+0000 mon.b (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.147460+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:59:24.413 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.150703+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.151926+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.155:0/3647512796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.160414+0000 mon.a (mon.0) 558 : cluster [INF] Active manager daemon y restarted 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.160952+0000 mon.a (mon.0) 559 : cluster [INF] Activating manager daemon y 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.171739+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.172142+0000 mon.a (mon.0) 561 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0113494s), standbys: x 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.173584+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.173801+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.173998+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.175155+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:59:24.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.175385+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.175622+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.176001+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.176319+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.176653+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.176955+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.177204+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.177399+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.177665+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.177938+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.178156+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:59:24.415 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27513]: audit 2023-10-09T05:59:24.178740+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:59:24.416 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: debug 2023-10-09T05:59:24.136+0000 7f03b9cdb680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:59:24.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: [09/Oct/2023:05:59:24] ENGINE Bus STARTING 2023-10-09T05:59:24.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: Warning: Permanently added '172.21.15.125' (ECDSA) to the list of known hosts. 2023-10-09T05:59:24.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: Warning: Permanently added '172.21.15.155' (ECDSA) to the list of known hosts. 2023-10-09T05:59:24.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: CherryPy Checker: 2023-10-09T05:59:24.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: The Application mounted at '' has an empty config. 2023-10-09T05:59:24.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: [09/Oct/2023:05:59:24] ENGINE Serving on http://:::9283 2023-10-09T05:59:24.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:24 smithi125 bash[27779]: [09/Oct/2023:05:59:24] ENGINE Bus STARTED 2023-10-09T05:59:25.226 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled node-exporter update... 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: cluster 2023-10-09T05:59:24.203079+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.341617+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.344459+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.407875+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.427505+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.428117+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.435123+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:25.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:25 smithi155 bash[27817]: audit 2023-10-09T05:59:24.435600+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:25.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: cluster 2023-10-09T05:59:24.203079+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.341617+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.344459+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.407875+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.427505+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.428117+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.435123+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[33919]: audit 2023-10-09T05:59:24.435600+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: cluster 2023-10-09T05:59:24.203079+0000 mon.a (mon.0) 562 : cluster [INF] Manager daemon y is now available 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.341617+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.344459+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.407875+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.427505+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.428117+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.435123+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:25 smithi125 bash[27513]: audit 2023-10-09T05:59:24.435600+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:59:26.007 DEBUG:teuthology.orchestra.run.smithi125:node-exporter.a> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T05:59:26.010 DEBUG:teuthology.orchestra.run.smithi155:node-exporter.b> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T05:59:26.012 INFO:tasks.cephadm:Adding alertmanager.a on smithi125 2023-10-09T05:59:26.012 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi125=a' 2023-10-09T05:59:26.041 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:59:26.126 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:59:26.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[27513]: cluster 2023-10-09T05:59:25.212302+0000 mon.a (mon.0) 566 : cluster [DBG] mgrmap e17: y(active, since 1.05151s), standbys: x 2023-10-09T05:59:26.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[27513]: cephadm 2023-10-09T05:59:25.219283+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T05:59:26.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[27513]: audit 2023-10-09T05:59:25.224070+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:26.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[33919]: cluster 2023-10-09T05:59:25.212302+0000 mon.a (mon.0) 566 : cluster [DBG] mgrmap e17: y(active, since 1.05151s), standbys: x 2023-10-09T05:59:26.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[33919]: cephadm 2023-10-09T05:59:25.219283+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T05:59:26.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:26 smithi125 bash[33919]: audit 2023-10-09T05:59:25.224070+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:26 smithi155 bash[27817]: cluster 2023-10-09T05:59:25.212302+0000 mon.a (mon.0) 566 : cluster [DBG] mgrmap e17: y(active, since 1.05151s), standbys: x 2023-10-09T05:59:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:26 smithi155 bash[27817]: cephadm 2023-10-09T05:59:25.219283+0000 mgr.y (mgr.14430) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi125=a;smithi155=b;count:2 2023-10-09T05:59:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:26 smithi155 bash[27817]: audit 2023-10-09T05:59:25.224070+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:27 smithi125 bash[27513]: cluster 2023-10-09T05:59:26.177086+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:27.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:27 smithi125 bash[27513]: cluster 2023-10-09T05:59:26.337008+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-10-09T05:59:27.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:27 smithi125 bash[33919]: cluster 2023-10-09T05:59:26.177086+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:27 smithi125 bash[33919]: cluster 2023-10-09T05:59:26.337008+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-10-09T05:59:27.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:27 smithi155 bash[27817]: cluster 2023-10-09T05:59:26.177086+0000 mgr.y (mgr.14430) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:27 smithi155 bash[27817]: cluster 2023-10-09T05:59:26.337008+0000 mon.a (mon.0) 568 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-10-09T05:59:29.142 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled alertmanager update... 2023-10-09T05:59:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:29 smithi125 bash[27513]: cluster 2023-10-09T05:59:28.177716+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:29.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:29 smithi125 bash[27513]: audit 2023-10-09T05:59:29.139198+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:29.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:29 smithi125 bash[33919]: cluster 2023-10-09T05:59:28.177716+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:29.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:29 smithi125 bash[33919]: audit 2023-10-09T05:59:29.139198+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:29.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:29 smithi155 bash[27817]: cluster 2023-10-09T05:59:28.177716+0000 mgr.y (mgr.14430) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:29.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:29 smithi155 bash[27817]: audit 2023-10-09T05:59:29.139198+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:29.865 DEBUG:teuthology.orchestra.run.smithi125:alertmanager.a> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T05:59:29.868 INFO:tasks.cephadm:Adding grafana.a on smithi155 2023-10-09T05:59:29.868 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi155=a' 2023-10-09T05:59:30.077 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:59:30.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[33919]: audit 2023-10-09T05:59:29.128522+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:30.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[33919]: cephadm 2023-10-09T05:59:29.131986+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T05:59:30.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[33919]: audit 2023-10-09T05:59:29.977348+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:30.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[27513]: audit 2023-10-09T05:59:29.128522+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:30.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[27513]: cephadm 2023-10-09T05:59:29.131986+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T05:59:30.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:30 smithi125 bash[27513]: audit 2023-10-09T05:59:29.977348+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:30 smithi155 bash[27817]: audit 2023-10-09T05:59:29.128522+0000 mgr.y (mgr.14430) 6 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi125=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:30 smithi155 bash[27817]: cephadm 2023-10-09T05:59:29.131986+0000 mgr.y (mgr.14430) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi125=a;count:1 2023-10-09T05:59:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:30 smithi155 bash[27817]: audit 2023-10-09T05:59:29.977348+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: cluster 2023-10-09T05:59:30.178292+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:30.771209+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:31.054208+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:31.056447+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:31.117867+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:31.166927+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[27513]: audit 2023-10-09T05:59:31.173908+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: cluster 2023-10-09T05:59:30.178292+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:30.771209+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:31.054208+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:31.056447+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:31.117867+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:31.166927+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:31 smithi125 bash[33919]: audit 2023-10-09T05:59:31.173908+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: cluster 2023-10-09T05:59:30.178292+0000 mgr.y (mgr.14430) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:30.771209+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:31.054208+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:31.056447+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:31.117867+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:31.166927+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:31 smithi155 bash[27817]: audit 2023-10-09T05:59:31.173908+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:32.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[27513]: cephadm 2023-10-09T05:59:31.059428+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:59:32.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[27513]: cephadm 2023-10-09T05:59:31.060092+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[27513]: cephadm 2023-10-09T05:59:31.082472+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[27513]: cephadm 2023-10-09T05:59:31.099592+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[27513]: cephadm 2023-10-09T05:59:31.198313+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi155 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[33919]: cephadm 2023-10-09T05:59:31.059428+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[33919]: cephadm 2023-10-09T05:59:31.060092+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[33919]: cephadm 2023-10-09T05:59:31.082472+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[33919]: cephadm 2023-10-09T05:59:31.099592+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:32 smithi125 bash[33919]: cephadm 2023-10-09T05:59:31.198313+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi155 2023-10-09T05:59:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:32 smithi155 bash[27817]: cephadm 2023-10-09T05:59:31.059428+0000 mgr.y (mgr.14430) 9 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T05:59:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:32 smithi155 bash[27817]: cephadm 2023-10-09T05:59:31.060092+0000 mgr.y (mgr.14430) 10 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T05:59:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:32 smithi155 bash[27817]: cephadm 2023-10-09T05:59:31.082472+0000 mgr.y (mgr.14430) 11 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:32 smithi155 bash[27817]: cephadm 2023-10-09T05:59:31.099592+0000 mgr.y (mgr.14430) 12 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:59:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:32 smithi155 bash[27817]: cephadm 2023-10-09T05:59:31.198313+0000 mgr.y (mgr.14430) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi155 2023-10-09T05:59:33.509 INFO:teuthology.orchestra.run.smithi155.stdout:Scheduled grafana update... 2023-10-09T05:59:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:33 smithi125 bash[27513]: cluster 2023-10-09T05:59:32.178980+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:33 smithi125 bash[33919]: cluster 2023-10-09T05:59:32.178980+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:33.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:33 smithi155 bash[27817]: cluster 2023-10-09T05:59:32.178980+0000 mgr.y (mgr.14430) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[27513]: audit 2023-10-09T05:59:33.497327+0000 mgr.y (mgr.14430) 15 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[27513]: cephadm 2023-10-09T05:59:33.500299+0000 mgr.y (mgr.14430) 16 : cephadm [INF] Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T05:59:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[27513]: audit 2023-10-09T05:59:33.507494+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[33919]: audit 2023-10-09T05:59:33.497327+0000 mgr.y (mgr.14430) 15 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[33919]: cephadm 2023-10-09T05:59:33.500299+0000 mgr.y (mgr.14430) 16 : cephadm [INF] Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T05:59:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:34 smithi125 bash[33919]: audit 2023-10-09T05:59:33.507494+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:34 smithi155 bash[27817]: audit 2023-10-09T05:59:33.497327+0000 mgr.y (mgr.14430) 15 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi155=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:34 smithi155 bash[27817]: cephadm 2023-10-09T05:59:33.500299+0000 mgr.y (mgr.14430) 16 : cephadm [INF] Saving service grafana spec with placement smithi155=a;count:1 2023-10-09T05:59:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:34 smithi155 bash[27817]: audit 2023-10-09T05:59:33.507494+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:35.368 DEBUG:teuthology.orchestra.run.smithi155:grafana.a> sudo journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T05:59:35.370 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T05:59:35.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:59:35.468 INFO:journalctl@ceph.grafana.a.smithi155.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-09T05:59:35.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:35 smithi125 bash[33919]: cluster 2023-10-09T05:59:34.179534+0000 mgr.y (mgr.14430) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:35.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:35 smithi125 bash[27513]: cluster 2023-10-09T05:59:34.179534+0000 mgr.y (mgr.14430) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:35 smithi155 bash[27817]: cluster 2023-10-09T05:59:34.179534+0000 mgr.y (mgr.14430) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:37.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:37 smithi125 bash[33919]: cluster 2023-10-09T05:59:36.180121+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:37.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:37 smithi125 bash[27513]: cluster 2023-10-09T05:59:36.180121+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:37 smithi155 bash[27817]: cluster 2023-10-09T05:59:36.180121+0000 mgr.y (mgr.14430) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:38.553 INFO:teuthology.orchestra.run.smithi125.stdout:[client.0] 2023-10-09T05:59:38.553 INFO:teuthology.orchestra.run.smithi125.stdout: key = AQDKliNlIAp0IBAA5aex4vBMxthyr7vsn8AJfQ== 2023-10-09T05:59:39.301 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T05:59:39.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T05:59:39.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T05:59:39.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:59:39.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[27513]: cluster 2023-10-09T05:59:38.180682+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:39.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[27513]: audit 2023-10-09T05:59:38.543701+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.125:0/2547053039' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[27513]: audit 2023-10-09T05:59:38.544301+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[27513]: audit 2023-10-09T05:59:38.551495+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[33919]: cluster 2023-10-09T05:59:38.180682+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[33919]: audit 2023-10-09T05:59:38.543701+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.125:0/2547053039' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[33919]: audit 2023-10-09T05:59:38.544301+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:39 smithi125 bash[33919]: audit 2023-10-09T05:59:38.551495+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:39.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:39 smithi155 bash[27817]: cluster 2023-10-09T05:59:38.180682+0000 mgr.y (mgr.14430) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:39.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:39 smithi155 bash[27817]: audit 2023-10-09T05:59:38.543701+0000 mon.c (mon.2) 37 : audit [INF] from='client.? 172.21.15.125:0/2547053039' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:39 smithi155 bash[27817]: audit 2023-10-09T05:59:38.544301+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:39.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:39 smithi155 bash[27817]: audit 2023-10-09T05:59:38.551495+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:41.552 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:41 smithi155 bash[27817]: cluster 2023-10-09T05:59:40.181301+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:41 smithi125 bash[27513]: cluster 2023-10-09T05:59:40.181301+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:41.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:41 smithi125 bash[33919]: cluster 2023-10-09T05:59:40.181301+0000 mgr.y (mgr.14430) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:43.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:43 smithi125 bash[27513]: cluster 2023-10-09T05:59:42.181966+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:43.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:43 smithi125 bash[33919]: cluster 2023-10-09T05:59:42.181966+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:43 smithi155 bash[27817]: cluster 2023-10-09T05:59:42.181966+0000 mgr.y (mgr.14430) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:44.480 INFO:teuthology.orchestra.run.smithi155.stdout:[client.1] 2023-10-09T05:59:44.480 INFO:teuthology.orchestra.run.smithi155.stdout: key = AQDQliNlDskhHBAAiFY5/ePLUQp+96Z/Ta0u8g== 2023-10-09T05:59:44.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:44 smithi155 bash[27817]: audit 2023-10-09T05:59:44.471061+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.155:0/2347008309' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:44 smithi155 bash[27817]: audit 2023-10-09T05:59:44.471634+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:44 smithi155 bash[27817]: audit 2023-10-09T05:59:44.478435+0000 mon.a (mon.0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:44.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[27513]: audit 2023-10-09T05:59:44.471061+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.155:0/2347008309' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[27513]: audit 2023-10-09T05:59:44.471634+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[27513]: audit 2023-10-09T05:59:44.478435+0000 mon.a (mon.0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[33919]: audit 2023-10-09T05:59:44.471061+0000 mon.b (mon.1) 22 : audit [INF] from='client.? 172.21.15.155:0/2347008309' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[33919]: audit 2023-10-09T05:59:44.471634+0000 mon.a (mon.0) 578 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:59:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:44 smithi125 bash[33919]: audit 2023-10-09T05:59:44.478435+0000 mon.a (mon.0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:59:45.546 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2023-10-09T05:59:45.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T05:59:45.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T05:59:45.564 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T05:59:45.564 DEBUG:teuthology.orchestra.run.smithi125:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T05:59:45.634 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:45 smithi155 systemd[1]: Started Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:59:45.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:45 smithi155 bash[27817]: cluster 2023-10-09T05:59:44.182597+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:45.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:45 smithi125 bash[27513]: cluster 2023-10-09T05:59:44.182597+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:45.853 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:45 smithi125 bash[33919]: cluster 2023-10-09T05:59:44.182597+0000 mgr.y (mgr.14430) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:45.872 INFO:teuthology.misc.health.smithi125.stdout: 2023-10-09T05:59:45.872 INFO:teuthology.misc.health.smithi125.stdout:{"epoch":51,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T05:59:24.160536+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:59:45.888 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T05:59:45.888 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T05:59:45.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T05:59:46.359 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:46 smithi125 systemd[1]: Started Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:59:46.620 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[52681]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T05:59:46.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: audit 2023-10-09T05:59:45.610367+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: cephadm 2023-10-09T05:59:45.613964+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi125 2023-10-09T05:59:46.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: audit 2023-10-09T05:59:45.870042+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.125:0/13219757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:46.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: cluster 2023-10-09T05:59:46.183158+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:46.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: audit 2023-10-09T05:59:46.339714+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[27817]: cephadm 2023-10-09T05:59:46.341216+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi155 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.659Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.659Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.659Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.659Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.659Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:59:46.833 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.661Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.661Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.662Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.668Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.668Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.735µs 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.668Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.669Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.669Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=49.594µs wal_replay_duration=349.59µs total_replay_duration=436.525µs 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.671Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.671Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T05:59:46.834 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.671Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:59:46.835 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.704Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=33.059477ms db_storage=849ns remote_storage=2.237µs web_handler=545ns query_engine=860ns scrape=381.231µs scrape_sd=28.91µs notify=1.227µs notify_sd=2.552µs rules=32.343689ms 2023-10-09T05:59:46.835 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 05:59:46 smithi155 bash[46921]: ts=2023-10-09T05:59:46.704Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T05:59:46.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: audit 2023-10-09T05:59:45.610367+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: cephadm 2023-10-09T05:59:45.613964+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi125 2023-10-09T05:59:46.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: audit 2023-10-09T05:59:45.870042+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.125:0/13219757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:46.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: cluster 2023-10-09T05:59:46.183158+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: audit 2023-10-09T05:59:46.339714+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[33919]: cephadm 2023-10-09T05:59:46.341216+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi155 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: audit 2023-10-09T05:59:45.610367+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: cephadm 2023-10-09T05:59:45.613964+0000 mgr.y (mgr.14430) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi125 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: audit 2023-10-09T05:59:45.870042+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.125:0/13219757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: cluster 2023-10-09T05:59:46.183158+0000 mgr.y (mgr.14430) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: audit 2023-10-09T05:59:46.339714+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:46 smithi125 bash[27513]: cephadm 2023-10-09T05:59:46.341216+0000 mgr.y (mgr.14430) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi155 2023-10-09T05:59:47.108 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:47 smithi155 systemd[1]: Started Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:59:47.393 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:47 smithi155 bash[47086]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-09T05:59:47.477 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T05:59:47.773 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: aa2a8d90b84c: Pulling fs layer 2023-10-09T05:59:47.774 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: b45d31ee2d7f: Pulling fs layer 2023-10-09T05:59:47.774 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: b5db1e299295: Pulling fs layer 2023-10-09T05:59:48.030 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: b45d31ee2d7f: Verifying Checksum 2023-10-09T05:59:48.030 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: b45d31ee2d7f: Download complete 2023-10-09T05:59:48.030 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:47 smithi125 bash[52681]: aa2a8d90b84c: Download complete 2023-10-09T05:59:48.075 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:47 smithi155 bash[47086]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-09T05:59:48.075 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: aa2a8d90b84c: Pulling fs layer 2023-10-09T05:59:48.075 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b45d31ee2d7f: Pulling fs layer 2023-10-09T05:59:48.097 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[52681]: b5db1e299295: Verifying Checksum 2023-10-09T05:59:48.097 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[52681]: b5db1e299295: Download complete 2023-10-09T05:59:48.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[27513]: audit 2023-10-09T05:59:47.093719+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:48.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[27513]: cephadm 2023-10-09T05:59:47.107848+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi125 2023-10-09T05:59:48.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[33919]: audit 2023-10-09T05:59:47.093719+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:48.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[33919]: cephadm 2023-10-09T05:59:47.107848+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi125 2023-10-09T05:59:48.393 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b5db1e299295: Pulling fs layer 2023-10-09T05:59:48.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[27817]: audit 2023-10-09T05:59:47.093719+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:48.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[27817]: cephadm 2023-10-09T05:59:47.107848+0000 mgr.y (mgr.14430) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi125 2023-10-09T05:59:48.730 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b45d31ee2d7f: Verifying Checksum 2023-10-09T05:59:48.730 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b45d31ee2d7f: Download complete 2023-10-09T05:59:48.730 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: aa2a8d90b84c: Verifying Checksum 2023-10-09T05:59:48.730 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: aa2a8d90b84c: Download complete 2023-10-09T05:59:48.730 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: aa2a8d90b84c: Pull complete 2023-10-09T05:59:48.988 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:48 smithi125 bash[52681]: aa2a8d90b84c: Pull complete 2023-10-09T05:59:49.102 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b5db1e299295: Verifying Checksum 2023-10-09T05:59:49.103 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:48 smithi155 bash[47086]: b5db1e299295: Download complete 2023-10-09T05:59:49.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:49 smithi125 bash[33919]: cluster 2023-10-09T05:59:48.183754+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:49.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:49 smithi125 bash[27513]: cluster 2023-10-09T05:59:48.183754+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:49 smithi155 bash[27817]: cluster 2023-10-09T05:59:48.183754+0000 mgr.y (mgr.14430) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:49.393 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:49 smithi155 bash[47086]: b45d31ee2d7f: Pull complete 2023-10-09T05:59:49.893 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:49 smithi155 bash[47086]: b5db1e299295: Pull complete 2023-10-09T05:59:49.893 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:49 smithi155 bash[47086]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T05:59:49.893 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:49 smithi155 bash[47086]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T05:59:50.385 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[52681]: b45d31ee2d7f: Pull complete 2023-10-09T05:59:50.762 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[27513]: audit 2023-10-09T05:59:49.429705+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:50.763 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[52681]: b5db1e299295: Pull complete 2023-10-09T05:59:50.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[33919]: audit 2023-10-09T05:59:49.429705+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:50.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[27817]: audit 2023-10-09T05:59:49.429705+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T05:59:51.135 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[52681]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-09T05:59:51.135 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:50 smithi125 bash[52681]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-09T05:59:51.143 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.923Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:59:51.143 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.923Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.924Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.924Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.925Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.925Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.925Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.925Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.925Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:59:51.144 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:59:51.145 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:59:51.146 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:59:51.147 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:59:51.148 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 05:59:50 smithi155 bash[47086]: ts=2023-10-09T05:59:50.926Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:59:51.458 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'rbd' created 2023-10-09T05:59:51.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:51 smithi155 bash[27817]: cluster 2023-10-09T05:59:50.184185+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:51.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:51 smithi155 bash[27817]: audit 2023-10-09T05:59:50.847280+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:59:51.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:51 smithi125 bash[33919]: cluster 2023-10-09T05:59:50.184185+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:51.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:51 smithi125 bash[33919]: audit 2023-10-09T05:59:50.847280+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:59:51.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:51 smithi125 bash[27513]: cluster 2023-10-09T05:59:50.184185+0000 mgr.y (mgr.14430) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:51.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:51 smithi125 bash[27513]: audit 2023-10-09T05:59:50.847280+0000 mon.a (mon.0) 584 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:59:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 05:59:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:05:59:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T05:59:52.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: audit 2023-10-09T05:59:51.452428+0000 mon.a (mon.0) 585 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:59:52.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: cluster 2023-10-09T05:59:51.452660+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:59:52.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: audit 2023-10-09T05:59:51.452843+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: audit 2023-10-09T05:59:51.454070+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: audit 2023-10-09T05:59:51.455631+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[27513]: audit 2023-10-09T05:59:51.456082+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: audit 2023-10-09T05:59:51.452428+0000 mon.a (mon.0) 585 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: cluster 2023-10-09T05:59:51.452660+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: audit 2023-10-09T05:59:51.452843+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: audit 2023-10-09T05:59:51.454070+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: audit 2023-10-09T05:59:51.455631+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:52.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:52 smithi125 bash[33919]: audit 2023-10-09T05:59:51.456082+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: audit 2023-10-09T05:59:51.452428+0000 mon.a (mon.0) 585 : audit [INF] from='client.? 172.21.15.125:0/3844453637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: cluster 2023-10-09T05:59:51.452660+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: audit 2023-10-09T05:59:51.452843+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: audit 2023-10-09T05:59:51.454070+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: audit 2023-10-09T05:59:51.455631+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:52.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:52 smithi155 bash[27817]: audit 2023-10-09T05:59:51.456082+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-09T05:59:53.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[27513]: cluster 2023-10-09T05:59:52.184740+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[27513]: audit 2023-10-09T05:59:52.463317+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[27513]: cluster 2023-10-09T05:59:52.463430+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[33919]: cluster 2023-10-09T05:59:52.184740+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[33919]: audit 2023-10-09T05:59:52.463317+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:59:53.801 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[33919]: cluster 2023-10-09T05:59:52.463430+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:59:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:53 smithi155 bash[27817]: cluster 2023-10-09T05:59:52.184740+0000 mgr.y (mgr.14430) 29 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:53 smithi155 bash[27817]: audit 2023-10-09T05:59:52.463317+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-09T05:59:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:53 smithi155 bash[27817]: cluster 2023-10-09T05:59:52.463430+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.928Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.928Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.928Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.928Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T05:59:54.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T05:59:54.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T05:59:54.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T05:59:54.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.929Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.930Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T05:59:54.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 05:59:53 smithi125 bash[52681]: ts=2023-10-09T05:59:53.930Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T05:59:54.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:54 smithi125 bash[33919]: cluster 2023-10-09T05:59:53.462207+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:59:54.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:54 smithi125 bash[27513]: cluster 2023-10-09T05:59:53.462207+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:59:54.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:54 smithi155 bash[27817]: cluster 2023-10-09T05:59:53.462207+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:59:55.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:55 smithi125 bash[27513]: cluster 2023-10-09T05:59:54.185339+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:55.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:55 smithi125 bash[33919]: cluster 2023-10-09T05:59:54.185339+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:55 smithi155 bash[27817]: cluster 2023-10-09T05:59:54.185339+0000 mgr.y (mgr.14430) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:57.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:57 smithi125 bash[27513]: cluster 2023-10-09T05:59:56.185851+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:57.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:57 smithi125 bash[33919]: cluster 2023-10-09T05:59:56.185851+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:57 smithi155 bash[27817]: cluster 2023-10-09T05:59:56.185851+0000 mgr.y (mgr.14430) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:59.081 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 05:59:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:05:59:58] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-10-09T05:59:59.737 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 05:59:59 smithi125 bash[27513]: cluster 2023-10-09T05:59:58.186244+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:59.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 05:59:59 smithi125 bash[33919]: cluster 2023-10-09T05:59:58.186244+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:59:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 05:59:59 smithi155 bash[27817]: cluster 2023-10-09T05:59:58.186244+0000 mgr.y (mgr.14430) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:00.502 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T06:00:00.752 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[27513]: audit 2023-10-09T05:59:59.533324+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.125:0/1153368108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.752 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[27513]: audit 2023-10-09T05:59:59.533889+0000 mon.a (mon.0) 592 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.752 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[27513]: cluster 2023-10-09T06:00:00.000170+0000 mon.a (mon.0) 593 : cluster [INF] overall HEALTH_OK 2023-10-09T06:00:00.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[33919]: audit 2023-10-09T05:59:59.533324+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.125:0/1153368108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.752 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[33919]: audit 2023-10-09T05:59:59.533889+0000 mon.a (mon.0) 592 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:00 smithi125 bash[33919]: cluster 2023-10-09T06:00:00.000170+0000 mon.a (mon.0) 593 : cluster [INF] overall HEALTH_OK 2023-10-09T06:00:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:00 smithi155 bash[27817]: audit 2023-10-09T05:59:59.533324+0000 mon.c (mon.2) 41 : audit [INF] from='client.? 172.21.15.125:0/1153368108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:00 smithi155 bash[27817]: audit 2023-10-09T05:59:59.533889+0000 mon.a (mon.0) 592 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:00:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:00 smithi155 bash[27817]: cluster 2023-10-09T06:00:00.000170+0000 mon.a (mon.0) 593 : cluster [INF] overall HEALTH_OK 2023-10-09T06:00:01.385 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:01 smithi125 systemd[1]: Started Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:01.599 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T06:00:01.600 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T06:00:01.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T06:00:01.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[27817]: cluster 2023-10-09T06:00:00.186800+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:01.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[27817]: audit 2023-10-09T06:00:00.496530+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:00:01.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[27817]: cluster 2023-10-09T06:00:00.496668+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T06:00:01.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[27817]: audit 2023-10-09T06:00:01.040687+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:01.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[27817]: audit 2023-10-09T06:00:01.043885+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[27513]: cluster 2023-10-09T06:00:00.186800+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[27513]: audit 2023-10-09T06:00:00.496530+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:00:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[27513]: cluster 2023-10-09T06:00:00.496668+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T06:00:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[27513]: audit 2023-10-09T06:00:01.040687+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[27513]: audit 2023-10-09T06:00:01.043885+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[33919]: cluster 2023-10-09T06:00:00.186800+0000 mgr.y (mgr.14430) 33 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[33919]: audit 2023-10-09T06:00:00.496530+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:00:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[33919]: cluster 2023-10-09T06:00:00.496668+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-09T06:00:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[33919]: audit 2023-10-09T06:00:01.040687+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:01 smithi125 bash[33919]: audit 2023-10-09T06:00:01.043885+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:01 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:02.635 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.387Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.387Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.391Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.125 port=9094 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.392Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.440Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.441Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.444Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T06:00:02.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:02 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:02.445Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T06:00:03.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:03 smithi125 bash[27513]: cluster 2023-10-09T06:00:02.187344+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:03.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:03 smithi125 bash[33919]: cluster 2023-10-09T06:00:02.187344+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:03 smithi155 bash[27817]: cluster 2023-10-09T06:00:02.187344+0000 mgr.y (mgr.14430) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:04.885 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:04 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:04.393Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000875754s 2023-10-09T06:00:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:05 smithi125 bash[27513]: cluster 2023-10-09T06:00:04.187939+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:05 smithi125 bash[27513]: audit 2023-10-09T06:00:04.445492+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:05 smithi125 bash[33919]: cluster 2023-10-09T06:00:04.187939+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:05 smithi125 bash[33919]: audit 2023-10-09T06:00:04.445492+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:05 smithi155 bash[27817]: cluster 2023-10-09T06:00:04.187939+0000 mgr.y (mgr.14430) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:05 smithi155 bash[27817]: audit 2023-10-09T06:00:04.445492+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:06.798 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:07.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27513]: cluster 2023-10-09T06:00:06.188739+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:07.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27513]: audit 2023-10-09T06:00:06.679567+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:07.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[33919]: cluster 2023-10-09T06:00:06.188739+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:07.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[33919]: audit 2023-10-09T06:00:06.679567+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:07 smithi155 bash[27817]: cluster 2023-10-09T06:00:06.188739+0000 mgr.y (mgr.14430) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:07 smithi155 bash[27817]: audit 2023-10-09T06:00:06.679567+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE Bus STOPPING 2023-10-09T06:00:08.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:00:08.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE Bus STOPPED 2023-10-09T06:00:08.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE Bus STARTING 2023-10-09T06:00:08.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE Serving on http://:::9283 2023-10-09T06:00:08.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:07 smithi125 bash[27779]: [09/Oct/2023:06:00:07] ENGINE Bus STARTED 2023-10-09T06:00:08.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.643371+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.645189+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:08.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.646405+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.653824+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.748160+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.753067+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.759381+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.760005+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: audit 2023-10-09T06:00:07.766011+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: cephadm 2023-10-09T06:00:07.788339+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi155 2023-10-09T06:00:08.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27513]: cluster 2023-10-09T06:00:08.189408+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:08] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.643371+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.645189+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.646405+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.653824+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.748160+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.753067+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.759381+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.760005+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: audit 2023-10-09T06:00:07.766011+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:08.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: cephadm 2023-10-09T06:00:07.788339+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi155 2023-10-09T06:00:08.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:08 smithi125 bash[33919]: cluster 2023-10-09T06:00:08.189408+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.643371+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.645189+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.646405+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:09.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.653824+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.748160+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.753067+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.759381+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.760005+0000 mgr.y (mgr.14430) 37 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: audit 2023-10-09T06:00:07.766011+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: cephadm 2023-10-09T06:00:07.788339+0000 mgr.y (mgr.14430) 38 : cephadm [INF] Deploying daemon grafana.a on smithi155 2023-10-09T06:00:09.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:08 smithi155 bash[27817]: cluster 2023-10-09T06:00:08.189408+0000 mgr.y (mgr.14430) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:09.867 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:00:10.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:09 smithi125 bash[27513]: audit 2023-10-09T06:00:09.857135+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.125:0/3383397444' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:00:10.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:09 smithi125 bash[33919]: audit 2023-10-09T06:00:09.857135+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.125:0/3383397444' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:00:10.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:09 smithi155 bash[27817]: audit 2023-10-09T06:00:09.857135+0000 mon.a (mon.0) 604 : audit [DBG] from='client.? 172.21.15.125:0/3383397444' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:00:10.615 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":18,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":26344715},{"type":"v1","addr":"172.21.15.125:6801","nonce":26344715}]},"active_addr":"172.21.15.125:6801/26344715","active_change":"2023-10-09T05:59:24.160779+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":4016621680}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":457536542}]}]} 2023-10-09T06:00:10.619 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T06:00:10.619 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:00:10.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:00:11.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:11 smithi125 bash[27513]: cluster 2023-10-09T06:00:10.190030+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:11.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:11 smithi125 bash[33919]: cluster 2023-10-09T06:00:10.190030+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:11.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:11 smithi155 bash[27817]: cluster 2023-10-09T06:00:10.190030+0000 mgr.y (mgr.14430) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:12.885 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:12 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:12.396Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004170626s 2023-10-09T06:00:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:13 smithi125 bash[27513]: cluster 2023-10-09T06:00:12.190782+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:13.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:13 smithi125 bash[33919]: cluster 2023-10-09T06:00:12.190782+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:13 smithi155 bash[27817]: cluster 2023-10-09T06:00:12.190782+0000 mgr.y (mgr.14430) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:15 smithi125 bash[27513]: cluster 2023-10-09T06:00:14.191339+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:15 smithi125 bash[27513]: audit 2023-10-09T06:00:14.468551+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:15 smithi125 bash[33919]: cluster 2023-10-09T06:00:14.191339+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:15 smithi125 bash[33919]: audit 2023-10-09T06:00:14.468551+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:15 smithi155 bash[27817]: cluster 2023-10-09T06:00:14.191339+0000 mgr.y (mgr.14430) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:15 smithi155 bash[27817]: audit 2023-10-09T06:00:14.468551+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:17.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:17 smithi125 bash[33919]: cluster 2023-10-09T06:00:16.192077+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:17.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:17 smithi125 bash[27513]: cluster 2023-10-09T06:00:16.192077+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:17 smithi155 bash[27817]: cluster 2023-10-09T06:00:16.192077+0000 mgr.y (mgr.14430) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:18.535 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:00:18.535 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":55,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:00:00.485435+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:00:18.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:18] "GET /metrics HTTP/1.1" 200 155654 "" "Prometheus/2.33.4" 2023-10-09T06:00:19.325 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:00:19.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:00:19.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:19 smithi125 bash[27513]: cluster 2023-10-09T06:00:18.192696+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:19.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:19 smithi125 bash[27513]: audit 2023-10-09T06:00:18.531498+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.125:0/908307537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:19 smithi125 bash[33919]: cluster 2023-10-09T06:00:18.192696+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:19 smithi125 bash[33919]: audit 2023-10-09T06:00:18.531498+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.125:0/908307537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:19 smithi155 bash[27817]: cluster 2023-10-09T06:00:18.192696+0000 mgr.y (mgr.14430) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:19 smithi155 bash[27817]: audit 2023-10-09T06:00:18.531498+0000 mon.a (mon.0) 606 : audit [DBG] from='client.? 172.21.15.125:0/908307537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:21.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:21 smithi155 bash[27817]: cluster 2023-10-09T06:00:20.193291+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:21.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:21 smithi125 bash[27513]: cluster 2023-10-09T06:00:20.193291+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:21.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:21 smithi125 bash[33919]: cluster 2023-10-09T06:00:20.193291+0000 mgr.y (mgr.14430) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:23.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:23 smithi125 bash[27513]: cluster 2023-10-09T06:00:22.194109+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:23.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:23 smithi125 bash[33919]: cluster 2023-10-09T06:00:22.194109+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:23 smithi155 bash[27817]: cluster 2023-10-09T06:00:22.194109+0000 mgr.y (mgr.14430) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:24.101 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:25.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:25 smithi125 bash[33919]: cluster 2023-10-09T06:00:24.194698+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:25.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:25 smithi125 bash[27513]: cluster 2023-10-09T06:00:24.194698+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:25 smithi155 bash[27817]: cluster 2023-10-09T06:00:24.194698+0000 mgr.y (mgr.14430) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:27.147 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:00:27.147 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":55,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:00:00.485435+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:00:27.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:27 smithi125 bash[27513]: cluster 2023-10-09T06:00:26.195485+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:27.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:27 smithi125 bash[27513]: audit 2023-10-09T06:00:27.143992+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.125:0/2761449471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:27 smithi125 bash[33919]: cluster 2023-10-09T06:00:26.195485+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:27 smithi125 bash[33919]: audit 2023-10-09T06:00:27.143992+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.125:0/2761449471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:27 smithi155 bash[27817]: cluster 2023-10-09T06:00:26.195485+0000 mgr.y (mgr.14430) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:27 smithi155 bash[27817]: audit 2023-10-09T06:00:27.143992+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.125:0/2761449471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:00:27.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:00:27.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:00:27.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:00:27.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:00:27.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:00:27.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:00:27.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:00:27.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:00:29.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:28] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-10-09T06:00:29.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:29 smithi125 bash[27513]: cluster 2023-10-09T06:00:28.195826+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:29 smithi125 bash[33919]: cluster 2023-10-09T06:00:28.195826+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:29 smithi155 bash[27817]: cluster 2023-10-09T06:00:28.195826+0000 mgr.y (mgr.14430) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:30.261 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:30 smithi155 systemd[1]: Started Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:31.405 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[27817]: cluster 2023-10-09T06:00:30.196322+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:31.405 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[27817]: audit 2023-10-09T06:00:30.237501+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:31.405 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[27817]: audit 2023-10-09T06:00:30.239818+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:31.405 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T06:00:31.405 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T06:00:31.405 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T06:00:31.406 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-09T06:00:31.407 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-09T06:00:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[27513]: cluster 2023-10-09T06:00:30.196322+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:31.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[27513]: audit 2023-10-09T06:00:30.237501+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[27513]: audit 2023-10-09T06:00:30.239818+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[33919]: cluster 2023-10-09T06:00:30.196322+0000 mgr.y (mgr.14430) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[33919]: audit 2023-10-09T06:00:30.237501+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:31 smithi125 bash[33919]: audit 2023-10-09T06:00:30.239818+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-09T06:00:31.657 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T06:00:31.943 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:31.943 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-09T06:00:31.943 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-09T06:00:31.943 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-09T06:00:31.944 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:31 smithi155 bash[47697]: t=2023-10-09T06:00:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-09T06:00:32.204 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-09T06:00:32.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-09T06:00:32.205 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-09T06:00:32.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T06:00:32.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-09T06:00:32.464 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-09T06:00:32.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-09T06:00:32.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-09T06:00:32.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-09T06:00:32.465 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-09T06:00:32.726 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-09T06:00:32.727 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-09T06:00:32.727 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T06:00:32.774 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.777 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.778 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.780 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.781 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.788 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.788 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.788 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:32.978 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-09T06:00:32.978 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-09T06:00:32.978 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-09T06:00:32.978 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-09T06:00:32.978 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-09T06:00:32.979 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:32 smithi155 bash[47697]: t=2023-10-09T06:00:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T06:00:33.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-09T06:00:33.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-09T06:00:33.499 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[27817]: cluster 2023-10-09T06:00:32.196989+0000 mgr.y (mgr.14430) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-09T06:00:33.500 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-09T06:00:33.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:33 smithi125 bash[27513]: cluster 2023-10-09T06:00:32.196989+0000 mgr.y (mgr.14430) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:33.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:33 smithi125 bash[33919]: cluster 2023-10-09T06:00:32.196989+0000 mgr.y (mgr.14430) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-09T06:00:33.769 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-09T06:00:33.770 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-09T06:00:33.770 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-09T06:00:33.770 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-09T06:00:34.046 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:33 smithi155 bash[47697]: t=2023-10-09T06:00:33+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T06:00:34.047 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-09T06:00:34.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-09T06:00:34.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-09T06:00:34.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-09T06:00:34.300 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-09T06:00:34.300 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-09T06:00:34.300 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-09T06:00:34.300 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-09T06:00:34.300 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-09T06:00:34.568 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-09T06:00:34.569 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-09T06:00:34.829 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-09T06:00:34.830 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-09T06:00:35.115 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-09T06:00:35.115 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-09T06:00:35.116 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-09T06:00:35.116 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:34 smithi155 bash[47697]: t=2023-10-09T06:00:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-09T06:00:35.116 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-09T06:00:35.116 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-09T06:00:35.116 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-09T06:00:35.377 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T06:00:35.377 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-09T06:00:35.377 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-09T06:00:35.378 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-09T06:00:35.378 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-09T06:00:35.378 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-09T06:00:35.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:35 smithi125 bash[27513]: cluster 2023-10-09T06:00:34.197573+0000 mgr.y (mgr.14430) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:35.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:35 smithi125 bash[27513]: audit 2023-10-09T06:00:34.488139+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:35.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:35 smithi125 bash[33919]: cluster 2023-10-09T06:00:34.197573+0000 mgr.y (mgr.14430) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:35 smithi125 bash[33919]: audit 2023-10-09T06:00:34.488139+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:35.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[27817]: cluster 2023-10-09T06:00:34.197573+0000 mgr.y (mgr.14430) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:35.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[27817]: audit 2023-10-09T06:00:34.488139+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-09T06:00:35.637 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-09T06:00:35.638 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-09T06:00:35.638 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-09T06:00:35.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T06:00:35.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T06:00:35.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T06:00:35.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T06:00:35.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-09T06:00:35.894 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:35 smithi155 bash[47697]: t=2023-10-09T06:00:35+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T06:00:36.184 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-09T06:00:36.444 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-09T06:00:36.444 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-09T06:00:36.444 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-09T06:00:36.445 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-09T06:00:36.445 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-09T06:00:36.445 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-09T06:00:36.445 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-09T06:00:36.706 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[27817]: audit 2023-10-09T06:00:35.543260+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.706 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[27817]: audit 2023-10-09T06:00:36.041972+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.706 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[27817]: audit 2023-10-09T06:00:36.327419+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:36.706 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[27817]: audit 2023-10-09T06:00:36.328726+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:36.706 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[27817]: audit 2023-10-09T06:00:36.336751+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-09T06:00:36.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-09T06:00:36.706 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-09T06:00:36.707 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-09T06:00:36.773 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[33919]: audit 2023-10-09T06:00:35.543260+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.773 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[33919]: audit 2023-10-09T06:00:36.041972+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.773 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[33919]: audit 2023-10-09T06:00:36.327419+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:36.773 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[33919]: audit 2023-10-09T06:00:36.328726+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:36.773 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[33919]: audit 2023-10-09T06:00:36.336751+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[27513]: audit 2023-10-09T06:00:35.543260+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[27513]: audit 2023-10-09T06:00:36.041972+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[27513]: audit 2023-10-09T06:00:36.327419+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:36.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[27513]: audit 2023-10-09T06:00:36.328726+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:36.774 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:36 smithi125 bash[27513]: audit 2023-10-09T06:00:36.336751+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:36.966 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-09T06:00:36.966 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-09T06:00:36.967 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:36 smithi155 bash[47697]: t=2023-10-09T06:00:36+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T06:00:37.235 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-09T06:00:37.236 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-09T06:00:37.505 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T06:00:37.506 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-09T06:00:37.506 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-09T06:00:37.506 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-09T06:00:37.774 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[27817]: cluster 2023-10-09T06:00:36.198208+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:37.774 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[27817]: cephadm 2023-10-09T06:00:36.344111+0000 mgr.y (mgr.14430) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:00:37.774 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[27817]: cephadm 2023-10-09T06:00:36.347559+0000 mgr.y (mgr.14430) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-09T06:00:37.775 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-09T06:00:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[33919]: cluster 2023-10-09T06:00:36.198208+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[33919]: cephadm 2023-10-09T06:00:36.344111+0000 mgr.y (mgr.14430) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:00:37.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[33919]: cephadm 2023-10-09T06:00:36.347559+0000 mgr.y (mgr.14430) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:00:37.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[27513]: cluster 2023-10-09T06:00:36.198208+0000 mgr.y (mgr.14430) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:37.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[27513]: cephadm 2023-10-09T06:00:36.344111+0000 mgr.y (mgr.14430) 54 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:00:37.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:37 smithi125 bash[27513]: cephadm 2023-10-09T06:00:36.347559+0000 mgr.y (mgr.14430) 55 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:00:38.035 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-09T06:00:38.035 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-09T06:00:38.035 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:37 smithi155 bash[47697]: t=2023-10-09T06:00:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-09T06:00:38.036 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-09T06:00:38.298 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T06:00:38.298 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T06:00:38.298 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T06:00:38.298 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-09T06:00:38.298 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T06:00:38.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-09T06:00:38.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-09T06:00:38.299 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-09T06:00:38.558 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-09T06:00:38.559 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-09T06:00:38.820 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-09T06:00:38.820 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-09T06:00:38.821 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:38 smithi155 bash[47697]: t=2023-10-09T06:00:38+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-09T06:00:39.090 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-09T06:00:39.091 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-09T06:00:39.093 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:38 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:38] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-10-09T06:00:39.367 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-09T06:00:39.367 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-09T06:00:39.367 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T06:00:39.368 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-09T06:00:39.368 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-09T06:00:39.368 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-09T06:00:39.368 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-09T06:00:39.368 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-09T06:00:39.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:39 smithi125 bash[27513]: cluster 2023-10-09T06:00:38.198742+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:39.628 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[27817]: cluster 2023-10-09T06:00:38.198742+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-09T06:00:39.629 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-09T06:00:39.630 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-09T06:00:39.630 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-09T06:00:39.630 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-09T06:00:39.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:39 smithi125 bash[33919]: cluster 2023-10-09T06:00:38.198742+0000 mgr.y (mgr.14430) 56 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-09T06:00:39.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-09T06:00:39.894 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:39 smithi155 bash[47697]: t=2023-10-09T06:00:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-09T06:00:40.168 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-09T06:00:40.169 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-09T06:00:40.169 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-09T06:00:40.437 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-09T06:00:40.437 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-09T06:00:40.437 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-09T06:00:40.437 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-09T06:00:40.438 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-09T06:00:40.438 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-09T06:00:40.438 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-09T06:00:40.690 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-09T06:00:40.690 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-09T06:00:40.690 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-09T06:00:40.690 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-09T06:00:40.690 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-09T06:00:40.691 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-09T06:00:40.691 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-09T06:00:40.691 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-09T06:00:40.691 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T06:00:40.959 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-09T06:00:40.959 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-09T06:00:40.959 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-09T06:00:40.959 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T06:00:40.959 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-09T06:00:40.960 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-09T06:00:40.960 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-09T06:00:40.960 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-09T06:00:41.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:40 smithi155 bash[47697]: t=2023-10-09T06:00:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T06:00:41.245 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T06:00:41.246 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-09T06:00:41.506 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-09T06:00:41.507 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T06:00:41.767 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[27817]: cluster 2023-10-09T06:00:40.199137+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-09T06:00:41.768 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T06:00:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:41 smithi125 bash[27513]: cluster 2023-10-09T06:00:40.199137+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:41.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:41 smithi125 bash[33919]: cluster 2023-10-09T06:00:40.199137+0000 mgr.y (mgr.14430) 57 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:42.028 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:41 smithi155 bash[47697]: t=2023-10-09T06:00:41+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-09T06:00:42.029 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-09T06:00:42.290 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T06:00:42.290 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-09T06:00:42.290 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-09T06:00:42.290 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-09T06:00:42.291 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-09T06:00:42.551 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-09T06:00:42.812 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-09T06:00:42.813 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:42 smithi155 bash[47697]: t=2023-10-09T06:00:42+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-09T06:00:43.064 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-09T06:00:43.065 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T06:00:43.316 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-09T06:00:43.316 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T06:00:43.317 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-09T06:00:43.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[27817]: cluster 2023-10-09T06:00:42.199812+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:43.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-09T06:00:43.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-09T06:00:43.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.106057056s 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T06:00:43.644 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T06:00:43.837 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:43 smithi125 bash[27513]: cluster 2023-10-09T06:00:42.199812+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:43.837 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:43 smithi125 bash[33919]: cluster 2023-10-09T06:00:42.199812+0000 mgr.y (mgr.14430) 58 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:44.143 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T06:00:44.143 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T06:00:44.143 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T06:00:44.143 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T06:00:44.143 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:00:43 smithi155 bash[47697]: t=2023-10-09T06:00:43+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T06:00:45.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:45 smithi125 bash[33919]: cluster 2023-10-09T06:00:44.200197+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:45.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:45 smithi125 bash[27513]: cluster 2023-10-09T06:00:44.200197+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:45 smithi155 bash[27817]: cluster 2023-10-09T06:00:44.200197+0000 mgr.y (mgr.14430) 59 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:47.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:47 smithi125 bash[33919]: cluster 2023-10-09T06:00:46.200645+0000 mgr.y (mgr.14430) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:47.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:47 smithi125 bash[27513]: cluster 2023-10-09T06:00:46.200645+0000 mgr.y (mgr.14430) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:47 smithi155 bash[27817]: cluster 2023-10-09T06:00:46.200645+0000 mgr.y (mgr.14430) 60 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:48.024 INFO:teuthology.orchestra.run.smithi125.stdout:180388626464 2023-10-09T06:00:48.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:00:49.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:48] "GET /metrics HTTP/1.1" 200 155656 "" "Prometheus/2.33.4" 2023-10-09T06:00:49.190 INFO:teuthology.orchestra.run.smithi125.stdout:128849018931 2023-10-09T06:00:49.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:00:49.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:49 smithi125 bash[27513]: cluster 2023-10-09T06:00:48.200980+0000 mgr.y (mgr.14430) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:49.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:49 smithi125 bash[33919]: cluster 2023-10-09T06:00:48.200980+0000 mgr.y (mgr.14430) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:49 smithi155 bash[27817]: cluster 2023-10-09T06:00:48.200980+0000 mgr.y (mgr.14430) 61 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:51.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:51 smithi125 bash[27513]: cluster 2023-10-09T06:00:50.201377+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:51.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:51 smithi125 bash[33919]: cluster 2023-10-09T06:00:50.201377+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:51.859 INFO:teuthology.orchestra.run.smithi125.stdout:55834574926 2023-10-09T06:00:51.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:00:51.864 INFO:teuthology.orchestra.run.smithi125.stdout:107374182459 2023-10-09T06:00:51.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:00:51.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:51 smithi155 bash[27817]: cluster 2023-10-09T06:00:50.201377+0000 mgr.y (mgr.14430) 62 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:00:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:00:52.283 INFO:teuthology.orchestra.run.smithi125.stdout:206158430233 2023-10-09T06:00:52.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:00:53.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:53 smithi125 bash[27513]: cluster 2023-10-09T06:00:52.201737+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:53.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:53 smithi125 bash[33919]: cluster 2023-10-09T06:00:52.201737+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:53 smithi155 bash[27817]: cluster 2023-10-09T06:00:52.201737+0000 mgr.y (mgr.14430) 63 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:54.116 INFO:teuthology.orchestra.run.smithi125.stdout:34359738455 2023-10-09T06:00:54.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:00:54.224 INFO:teuthology.orchestra.run.smithi125.stdout:77309411398 2023-10-09T06:00:54.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:00:54.376 INFO:teuthology.orchestra.run.smithi125.stdout:154618822699 2023-10-09T06:00:54.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:00:54.635 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:54 smithi125 systemd[1]: Stopping Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:00:54.635 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:54 smithi125 bash[56708]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T06:00:54.635 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:54 smithi125 bash[53377]: level=info ts=2023-10-09T06:00:54.611Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:00:55.468 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[56744]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:00:55.468 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[56907]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:00:55.468 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:55 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:00:55.468 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:55 smithi125 systemd[1]: Stopped Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:55.468 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:55 smithi125 systemd[1]: Started Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:55.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[27513]: cluster 2023-10-09T06:00:54.202106+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:55.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[27513]: audit 2023-10-09T06:00:55.446452+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:55.850 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[33919]: cluster 2023-10-09T06:00:54.202106+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:55.850 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:55 smithi125 bash[33919]: audit 2023-10-09T06:00:55.446452+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:55 smithi155 bash[27817]: cluster 2023-10-09T06:00:54.202106+0000 mgr.y (mgr.14430) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:55 smithi155 bash[27817]: audit 2023-10-09T06:00:55.446452+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:00:56.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[27513]: cephadm 2023-10-09T06:00:55.449861+0000 mgr.y (mgr.14430) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:00:56.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[27513]: cephadm 2023-10-09T06:00:55.455699+0000 mgr.y (mgr.14430) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:00:56.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[33919]: cephadm 2023-10-09T06:00:55.449861+0000 mgr.y (mgr.14430) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:00:56.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[33919]: cephadm 2023-10-09T06:00:55.455699+0000 mgr.y (mgr.14430) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:00:56.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.589Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T06:00:56.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.589Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.592Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.125 port=9094 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.594Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.631Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.631Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.636Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T06:00:56.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:56 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:56.636Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T06:00:56.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:56 smithi155 bash[27817]: cephadm 2023-10-09T06:00:55.449861+0000 mgr.y (mgr.14430) 65 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:00:56.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:56 smithi155 bash[27817]: cephadm 2023-10-09T06:00:55.455699+0000 mgr.y (mgr.14430) 66 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:00:56.922 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:56.924 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:56.924 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:56.930 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:56.930 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:57.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:57 smithi125 bash[27513]: cluster 2023-10-09T06:00:56.202902+0000 mgr.y (mgr.14430) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:57.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:57 smithi125 bash[33919]: cluster 2023-10-09T06:00:56.202902+0000 mgr.y (mgr.14430) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:57 smithi155 bash[27817]: cluster 2023-10-09T06:00:56.202902+0000 mgr.y (mgr.14430) 67 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:58.724 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:00:58 smithi125 bash[56935]: level=info ts=2023-10-09T06:00:58.594Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000769875s 2023-10-09T06:00:58.946 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:58.946 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:58.947 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:00:58.975 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:00:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:00:58] "GET /metrics HTTP/1.1" 200 155662 "" "Prometheus/2.33.4" 2023-10-09T06:00:59.875 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[27817]: cluster 2023-10-09T06:00:58.203484+0000 mgr.y (mgr.14430) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:59.875 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 systemd[1]: Stopping Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[48278]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.630Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.631Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:00:59.876 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.631Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T06:00:59.877 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.632Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:00:59.877 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.632Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T06:00:59.877 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:00:59 smithi155 bash[46921]: ts=2023-10-09T06:00:59.632Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T06:00:59.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:00:59 smithi125 bash[27513]: cluster 2023-10-09T06:00:58.203484+0000 mgr.y (mgr.14430) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:00:59.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:00:59 smithi125 bash[33919]: cluster 2023-10-09T06:00:58.203484+0000 mgr.y (mgr.14430) 68 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:00.492 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:00 smithi155 bash[48315]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:01:00.492 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:00 smithi155 bash[48376]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:01:00.492 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:00 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:01:00.492 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:00 smithi155 systemd[1]: Stopped Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:00.492 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:00 smithi155 systemd[1]: Started Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:00.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPING 2023-10-09T06:01:00.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:01:00.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPED 2023-10-09T06:01:00.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTING 2023-10-09T06:01:00.926 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Serving on http://:::9283 2023-10-09T06:01:00.926 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTED 2023-10-09T06:01:00.926 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPING 2023-10-09T06:01:00.926 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPED 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTING 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Serving on http://:::9283 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTED 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPING 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STOPPED 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTING 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Serving on http://:::9283 2023-10-09T06:01:00.927 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:00 smithi125 bash[27779]: [09/Oct/2023:06:01:00] ENGINE Bus STARTED 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.484Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.484Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.484Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.484Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.484Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T06:01:01.639 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.486Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.486Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.487Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.493Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.495Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.556381ms 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.495Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T06:01:01.640 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.637Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T06:01:01.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: cluster 2023-10-09T06:01:00.204108+0000 mgr.y (mgr.14430) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:01.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.462255+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.467860+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.468378+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.469662+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.470020+0000 mgr.y (mgr.14430) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.474701+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.487248+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.487800+0000 mgr.y (mgr.14430) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.490714+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.491238+0000 mgr.y (mgr.14430) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.496205+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.504597+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.505157+0000 mgr.y (mgr.14430) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.507551+0000 mon.c (mon.2) 55 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.508060+0000 mgr.y (mgr.14430) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.515090+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[27817]: audit 2023-10-09T06:01:00.519862+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: cluster 2023-10-09T06:01:00.204108+0000 mgr.y (mgr.14430) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.462255+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.467860+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.468378+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.469662+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.828 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.470020+0000 mgr.y (mgr.14430) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.474701+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.487248+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.487800+0000 mgr.y (mgr.14430) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.490714+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.491238+0000 mgr.y (mgr.14430) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.496205+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.504597+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.505157+0000 mgr.y (mgr.14430) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.829 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.507551+0000 mon.c (mon.2) 55 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.508060+0000 mgr.y (mgr.14430) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.515090+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[27513]: audit 2023-10-09T06:01:00.519862+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: cluster 2023-10-09T06:01:00.204108+0000 mgr.y (mgr.14430) 69 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.462255+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.467860+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.468378+0000 mgr.y (mgr.14430) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.469662+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.470020+0000 mgr.y (mgr.14430) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi125.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.474701+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.487248+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.487800+0000 mgr.y (mgr.14430) 72 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.490714+0000 mon.c (mon.2) 53 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.491238+0000 mgr.y (mgr.14430) 73 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi155.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.496205+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.504597+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.505157+0000 mgr.y (mgr.14430) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.507551+0000 mon.c (mon.2) 55 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.831 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.508060+0000 mgr.y (mgr.14430) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi155.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T06:01:01.832 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.515090+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:01.832 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:01 smithi125 bash[33919]: audit 2023-10-09T06:01:00.519862+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:01.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.637Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T06:01:01.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.637Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=89.455µs wal_replay_duration=142.833942ms total_replay_duration=144.515705ms 2023-10-09T06:01:01.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.640Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T06:01:01.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.640Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T06:01:01.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.640Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T06:01:01.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.668Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=27.321272ms db_storage=1.156µs remote_storage=2.132µs web_handler=490ns query_engine=1.074µs scrape=435.851µs scrape_sd=38.554µs notify=33.079µs notify_sd=11.78µs rules=26.252637ms 2023-10-09T06:01:01.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:01:01 smithi155 bash[48405]: ts=2023-10-09T06:01:01.668Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T06:01:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:03 smithi125 bash[33919]: cluster 2023-10-09T06:01:02.204700+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:03.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:03 smithi125 bash[27513]: cluster 2023-10-09T06:01:02.204700+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:03 smithi155 bash[27817]: cluster 2023-10-09T06:01:02.204700+0000 mgr.y (mgr.14430) 76 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:05 smithi125 bash[27513]: cluster 2023-10-09T06:01:04.205264+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:05 smithi125 bash[33919]: cluster 2023-10-09T06:01:04.205264+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:05 smithi155 bash[27817]: cluster 2023-10-09T06:01:04.205264+0000 mgr.y (mgr.14430) 77 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:06.885 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:01:06 smithi125 bash[56935]: level=info ts=2023-10-09T06:01:06.596Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002713853s 2023-10-09T06:01:07.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:07 smithi125 bash[27513]: cluster 2023-10-09T06:01:06.205953+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:07.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:07 smithi125 bash[27513]: audit 2023-10-09T06:01:06.407879+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:07 smithi125 bash[33919]: cluster 2023-10-09T06:01:06.205953+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:07 smithi125 bash[33919]: audit 2023-10-09T06:01:06.407879+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:07 smithi155 bash[27817]: cluster 2023-10-09T06:01:06.205953+0000 mgr.y (mgr.14430) 78 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:07 smithi155 bash[27817]: audit 2023-10-09T06:01:06.407879+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:09.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:08] "GET /metrics HTTP/1.1" 200 155662 "" "Prometheus/2.33.4" 2023-10-09T06:01:09.472 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:09 smithi125 bash[27513]: cluster 2023-10-09T06:01:08.206578+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:09.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:09 smithi125 bash[33919]: cluster 2023-10-09T06:01:08.206578+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:09.552 INFO:teuthology.orchestra.run.smithi125.stdout:206158430237 2023-10-09T06:01:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:09 smithi155 bash[27817]: cluster 2023-10-09T06:01:08.206578+0000 mgr.y (mgr.14430) 79 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:10.105 INFO:teuthology.orchestra.run.smithi125.stdout:107374182463 2023-10-09T06:01:10.748 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:10 smithi125 bash[33919]: audit 2023-10-09T06:01:09.550924+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.125:0/595273249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:01:10.748 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:10 smithi125 bash[33919]: audit 2023-10-09T06:01:10.103733+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.125:0/1914402767' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:01:10.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:10 smithi125 bash[27513]: audit 2023-10-09T06:01:09.550924+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.125:0/595273249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:01:10.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:10 smithi125 bash[27513]: audit 2023-10-09T06:01:10.103733+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.125:0/1914402767' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:01:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:10 smithi155 bash[27817]: audit 2023-10-09T06:01:09.550924+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.125:0/595273249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:01:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:10 smithi155 bash[27817]: audit 2023-10-09T06:01:10.103733+0000 mon.a (mon.0) 619 : audit [DBG] from='client.? 172.21.15.125:0/1914402767' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:01:11.580 INFO:teuthology.orchestra.run.smithi125.stdout:180388626469 2023-10-09T06:01:11.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:11 smithi125 bash[27513]: cluster 2023-10-09T06:01:10.206931+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:11.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:11 smithi125 bash[33919]: cluster 2023-10-09T06:01:10.206931+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:11 smithi155 bash[27817]: cluster 2023-10-09T06:01:10.206931+0000 mgr.y (mgr.14430) 80 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:01:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:01:12.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:12 smithi125 bash[27513]: audit 2023-10-09T06:01:11.579230+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.125:0/3948657355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:01:12.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:12 smithi125 bash[33919]: audit 2023-10-09T06:01:11.579230+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.125:0/3948657355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:01:12.712 INFO:teuthology.orchestra.run.smithi125.stdout:55834574931 2023-10-09T06:01:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:12 smithi155 bash[27817]: audit 2023-10-09T06:01:11.579230+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.125:0/3948657355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:01:13.275 INFO:teuthology.orchestra.run.smithi125.stdout:128849018936 2023-10-09T06:01:13.385 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430237 for osd.7 2023-10-09T06:01:13.386 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:13.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[27513]: cluster 2023-10-09T06:01:12.207553+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:13.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[27513]: audit 2023-10-09T06:01:12.711562+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.125:0/3732850971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:01:13.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[27513]: audit 2023-10-09T06:01:13.273725+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.125:0/471574919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:01:13.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[33919]: cluster 2023-10-09T06:01:12.207553+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:13.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[33919]: audit 2023-10-09T06:01:12.711562+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.125:0/3732850971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:01:13.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:13 smithi125 bash[33919]: audit 2023-10-09T06:01:13.273725+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.125:0/471574919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:01:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:13 smithi155 bash[27817]: cluster 2023-10-09T06:01:12.207553+0000 mgr.y (mgr.14430) 81 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:13 smithi155 bash[27817]: audit 2023-10-09T06:01:12.711562+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.125:0/3732850971' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:01:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:13 smithi155 bash[27817]: audit 2023-10-09T06:01:13.273725+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.125:0/471574919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:01:14.791 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626464 got 180388626469 for osd.6 2023-10-09T06:01:14.791 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:14.871 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182459 got 107374182463 for osd.3 2023-10-09T06:01:14.871 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:15.292 INFO:teuthology.orchestra.run.smithi125.stdout:77309411402 2023-10-09T06:01:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:15 smithi125 bash[27513]: cluster 2023-10-09T06:01:14.208213+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:15 smithi125 bash[27513]: audit 2023-10-09T06:01:15.291256+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.125:0/737457183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:01:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:15 smithi125 bash[33919]: cluster 2023-10-09T06:01:14.208213+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:15 smithi125 bash[33919]: audit 2023-10-09T06:01:15.291256+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.125:0/737457183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:01:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:15 smithi155 bash[27817]: cluster 2023-10-09T06:01:14.208213+0000 mgr.y (mgr.14430) 82 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:15 smithi155 bash[27817]: audit 2023-10-09T06:01:15.291256+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.125:0/737457183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:01:16.431 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018931 got 128849018936 for osd.4 2023-10-09T06:01:16.431 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:16.532 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574926 got 55834574931 for osd.1 2023-10-09T06:01:16.532 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:17.403 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411398 got 77309411402 for osd.2 2023-10-09T06:01:17.403 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:17.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:17 smithi125 bash[27513]: cluster 2023-10-09T06:01:16.208744+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:17.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:17 smithi125 bash[33919]: cluster 2023-10-09T06:01:16.208744+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:17.669 INFO:teuthology.orchestra.run.smithi125.stdout:34359738460 2023-10-09T06:01:17.690 INFO:teuthology.orchestra.run.smithi125.stdout:154618822704 2023-10-09T06:01:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:17 smithi155 bash[27817]: cluster 2023-10-09T06:01:16.208744+0000 mgr.y (mgr.14430) 83 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:18.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:18 smithi125 bash[27513]: audit 2023-10-09T06:01:17.667591+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.125:0/1177873050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:01:18.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:18 smithi125 bash[27513]: audit 2023-10-09T06:01:17.689174+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.125:0/2724365837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:01:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:18 smithi125 bash[33919]: audit 2023-10-09T06:01:17.667591+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.125:0/1177873050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:01:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:18 smithi125 bash[33919]: audit 2023-10-09T06:01:17.689174+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.125:0/2724365837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:01:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:18 smithi155 bash[27817]: audit 2023-10-09T06:01:17.667591+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.125:0/1177873050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:01:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:18 smithi155 bash[27817]: audit 2023-10-09T06:01:17.689174+0000 mon.a (mon.0) 621 : audit [DBG] from='client.? 172.21.15.125:0/2724365837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:01:19.126 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:18] "GET /metrics HTTP/1.1" 200 155657 "" "Prometheus/2.33.4" 2023-10-09T06:01:19.127 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822699 got 154618822704 for osd.5 2023-10-09T06:01:19.127 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:19.149 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738455 got 34359738460 for osd.0 2023-10-09T06:01:19.149 DEBUG:teuthology.parallel:result is None 2023-10-09T06:01:19.150 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:01:19.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: cluster 2023-10-09T06:01:18.209147+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: audit 2023-10-09T06:01:18.653915+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: audit 2023-10-09T06:01:18.656884+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: audit 2023-10-09T06:01:18.657886+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: audit 2023-10-09T06:01:18.665537+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[27513]: audit 2023-10-09T06:01:18.672260+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: cluster 2023-10-09T06:01:18.209147+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: audit 2023-10-09T06:01:18.653915+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: audit 2023-10-09T06:01:18.656884+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: audit 2023-10-09T06:01:18.657886+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: audit 2023-10-09T06:01:18.665537+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:19 smithi125 bash[33919]: audit 2023-10-09T06:01:18.672260+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: cluster 2023-10-09T06:01:18.209147+0000 mgr.y (mgr.14430) 84 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: audit 2023-10-09T06:01:18.653915+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: audit 2023-10-09T06:01:18.656884+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: audit 2023-10-09T06:01:18.657886+0000 mon.c (mon.2) 62 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: audit 2023-10-09T06:01:18.665537+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:19 smithi155 bash[27817]: audit 2023-10-09T06:01:18.672260+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:21.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:21 smithi155 bash[27817]: cluster 2023-10-09T06:01:20.209693+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:21.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:21 smithi125 bash[27513]: cluster 2023-10-09T06:01:20.209693+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:21.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:21 smithi125 bash[33919]: cluster 2023-10-09T06:01:20.209693+0000 mgr.y (mgr.14430) 85 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:01:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:01:23.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:23 smithi125 bash[27513]: cluster 2023-10-09T06:01:22.210396+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:23 smithi125 bash[33919]: cluster 2023-10-09T06:01:22.210396+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:23.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:23 smithi155 bash[27817]: cluster 2023-10-09T06:01:22.210396+0000 mgr.y (mgr.14430) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:23.962 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:25.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[27513]: cluster 2023-10-09T06:01:24.210937+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:25.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[27513]: audit 2023-10-09T06:01:24.542532+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:25.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[27513]: audit 2023-10-09T06:01:24.544704+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[27513]: audit 2023-10-09T06:01:24.545942+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[27513]: audit 2023-10-09T06:01:24.552313+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[33919]: cluster 2023-10-09T06:01:24.210937+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[33919]: audit 2023-10-09T06:01:24.542532+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[33919]: audit 2023-10-09T06:01:24.544704+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[33919]: audit 2023-10-09T06:01:24.545942+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:25 smithi125 bash[33919]: audit 2023-10-09T06:01:24.552313+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:25 smithi155 bash[27817]: cluster 2023-10-09T06:01:24.210937+0000 mgr.y (mgr.14430) 87 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:25 smithi155 bash[27817]: audit 2023-10-09T06:01:24.542532+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:25 smithi155 bash[27817]: audit 2023-10-09T06:01:24.544704+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:25 smithi155 bash[27817]: audit 2023-10-09T06:01:24.545942+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:25 smithi155 bash[27817]: audit 2023-10-09T06:01:24.552313+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:27.162 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:01:27.165 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:01:27.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:27 smithi125 bash[27513]: cluster 2023-10-09T06:01:26.211619+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:27 smithi125 bash[33919]: cluster 2023-10-09T06:01:26.211619+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:27 smithi155 bash[27817]: cluster 2023-10-09T06:01:26.211619+0000 mgr.y (mgr.14430) 88 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:27.907 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-10-09T06:01:26.211249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380972,"kb_used_data":4204,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580244,"statfs":{"total":767960285184,"available":765522169856,"internally_reserved":0,"allocated":4304896,"data_stored":1762545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.462366+0000","last_change":"2023-10-09T05:59:52.469538+0000","last_active":"2023-10-09T05:59:53.462366+0000","last_peered":"2023-10-09T05:59:53.462366+0000","last_clean":"2023-10-09T05:59:53.462366+0000","last_became_active":"2023-10-09T05:59:52.469011+0000","last_became_peered":"2023-10-09T05:59:52.469011+0000","last_unstale":"2023-10-09T05:59:53.462366+0000","last_undegraded":"2023-10-09T05:59:53.462366+0000","last_fullsized":"2023-10-09T05:59:53.462366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461573+0000","last_change":"2023-10-09T05:59:52.779228+0000","last_active":"2023-10-09T05:59:53.461573+0000","last_peered":"2023-10-09T05:59:53.461573+0000","last_clean":"2023-10-09T05:59:53.461573+0000","last_became_active":"2023-10-09T05:59:52.777994+0000","last_became_peered":"2023-10-09T05:59:52.777994+0000","last_unstale":"2023-10-09T05:59:53.461573+0000","last_undegraded":"2023-10-09T05:59:53.461573+0000","last_fullsized":"2023-10-09T05:59:53.461573+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630679+0000","last_change":"2023-10-09T05:59:52.778611+0000","last_active":"2023-10-09T05:59:54.630679+0000","last_peered":"2023-10-09T05:59:54.630679+0000","last_clean":"2023-10-09T05:59:54.630679+0000","last_became_active":"2023-10-09T05:59:52.777893+0000","last_became_peered":"2023-10-09T05:59:52.777893+0000","last_unstale":"2023-10-09T05:59:54.630679+0000","last_undegraded":"2023-10-09T05:59:54.630679+0000","last_fullsized":"2023-10-09T05:59:54.630679+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630490+0000","last_change":"2023-10-09T05:59:52.583913+0000","last_active":"2023-10-09T05:59:54.630490+0000","last_peered":"2023-10-09T05:59:54.630490+0000","last_clean":"2023-10-09T05:59:54.630490+0000","last_became_active":"2023-10-09T05:59:52.583298+0000","last_became_peered":"2023-10-09T05:59:52.583298+0000","last_unstale":"2023-10-09T05:59:54.630490+0000","last_undegraded":"2023-10-09T05:59:54.630490+0000","last_fullsized":"2023-10-09T05:59:54.630490+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630469+0000","last_change":"2023-10-09T05:58:52.535269+0000","last_active":"2023-10-09T05:59:54.630469+0000","last_peered":"2023-10-09T05:59:54.630469+0000","last_clean":"2023-10-09T05:59:54.630469+0000","last_became_active":"2023-10-09T05:58:52.534584+0000","last_became_peered":"2023-10-09T05:58:52.534584+0000","last_unstale":"2023-10-09T05:59:54.630469+0000","last_undegraded":"2023-10-09T05:59:54.630469+0000","last_fullsized":"2023-10-09T05:59:54.630469+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.471806+0000","last_change":"2023-10-09T05:59:52.469599+0000","last_active":"2023-10-09T05:59:54.471806+0000","last_peered":"2023-10-09T05:59:54.471806+0000","last_clean":"2023-10-09T05:59:54.471806+0000","last_became_active":"2023-10-09T05:59:52.469155+0000","last_became_peered":"2023-10-09T05:59:52.469155+0000","last_unstale":"2023-10-09T05:59:54.471806+0000","last_undegraded":"2023-10-09T05:59:54.471806+0000","last_fullsized":"2023-10-09T05:59:54.471806+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461109+0000","last_change":"2023-10-09T05:59:52.778084+0000","last_active":"2023-10-09T05:59:53.461109+0000","last_peered":"2023-10-09T05:59:53.461109+0000","last_clean":"2023-10-09T05:59:53.461109+0000","last_became_active":"2023-10-09T05:59:52.777374+0000","last_became_peered":"2023-10-09T05:59:52.777374+0000","last_unstale":"2023-10-09T05:59:53.461109+0000","last_undegraded":"2023-10-09T05:59:53.461109+0000","last_fullsized":"2023-10-09T05:59:53.461109+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.471782+0000","last_change":"2023-10-09T05:59:52.467577+0000","last_active":"2023-10-09T05:59:54.471782+0000","last_peered":"2023-10-09T05:59:54.471782+0000","last_clean":"2023-10-09T05:59:54.471782+0000","last_became_active":"2023-10-09T05:59:52.467143+0000","last_became_peered":"2023-10-09T05:59:52.467143+0000","last_unstale":"2023-10-09T05:59:54.471782+0000","last_undegraded":"2023-10-09T05:59:54.471782+0000","last_fullsized":"2023-10-09T05:59:54.471782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461366+0000","last_change":"2023-10-09T05:59:52.583128+0000","last_active":"2023-10-09T05:59:53.461366+0000","last_peered":"2023-10-09T05:59:53.461366+0000","last_clean":"2023-10-09T05:59:53.461366+0000","last_became_active":"2023-10-09T05:59:52.582692+0000","last_became_peered":"2023-10-09T05:59:52.582692+0000","last_unstale":"2023-10-09T05:59:53.461366+0000","last_undegraded":"2023-10-09T05:59:53.461366+0000","last_fullsized":"2023-10-09T05:59:53.461366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.86699999999999999,"15min":0.88100000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.0720000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.92900000000000005,"15min":0.93400000000000005},"min":{"1min":0.61399999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1859999999999999,"5min":1.452,"15min":1.452},"last":0.91400000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.879,"15min":0.88900000000000001},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.96899999999999997,"15min":0.98199999999999998},"min":{"1min":0.68300000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.125,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.73099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.89000000000000001,"15min":0.90100000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.1579999999999999,"5min":1.173,"15min":1.173},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.90300000000000002,"15min":0.90200000000000002},"min":{"1min":0.66100000000000003,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.3440000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.079}]},{"osd":3,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.95499999999999996,"15min":0.96499999999999997},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.2789999999999999,"5min":1.335,"15min":1.335},"last":1.04},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.95399999999999996,"15min":0.95599999999999996},"min":{"1min":0.64500000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.274,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.155}]},{"osd":4,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.94799999999999995,"15min":0.95799999999999996},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.2470000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.004},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.89400000000000002,"15min":0.89900000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2230000000000001,"5min":1.51,"15min":1.51},"last":0.83299999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.96799999999999997,"15min":0.98199999999999998},"min":{"1min":0.57899999999999996,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1659999999999999,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.873},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.999,"15min":1.0109999999999999},"min":{"1min":0.64800000000000002,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2729999999999999,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.119}]},{"osd":6,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.0389999999999999,"15min":1.048},"min":{"1min":0.66400000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3240000000000001,"5min":1.508,"15min":1.508},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.036,"15min":1.0409999999999999},"min":{"1min":0.69799999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3340000000000001,"5min":2.0019999999999998,"15min":2.0019999999999998},"last":1.1000000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626473,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.86499999999999999,"15min":0.88100000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1799999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86399999999999999,"15min":0.88600000000000001},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.3180000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.81399999999999995,"15min":0.82499999999999996},"min":{"1min":0.51900000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.0469999999999999,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.83199999999999996,"15min":0.81799999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0649999999999999,"5min":1.278,"15min":1.278},"last":1.2110000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.139,"5min":1.216,"15min":1.216},"last":1.091},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.93700000000000006,"15min":0.96499999999999997},"min":{"1min":0.66600000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2709999999999999,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":1.452}]},{"osd":3,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.86299999999999999,"15min":0.86099999999999999},"min":{"1min":0.622,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.105,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95099999999999996,"15min":0.94299999999999995},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.252,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.1499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.872,"15min":0.874},"min":{"1min":0.51100000000000001,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.21,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.3720000000000001},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.88900000000000001,"15min":0.88},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.222,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.048}]},{"osd":5,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.878,"15min":0.84399999999999997},"min":{"1min":0.56100000000000005,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3799999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.413},{"interface":"front","average":{"1min":1,"5min":0.93500000000000005,"15min":0.91100000000000003},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.0109999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:58 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.018,"15min":1.016},"min":{"1min":0.47399999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3480000000000001,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.4870000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.034,"15min":1.034},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.3819999999999999,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.2649999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.69899999999999995,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.36799999999999999},"max":{"1min":0.93799999999999994,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.68300000000000005,"15min":0.60699999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":0.93100000000000005,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.79700000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73299999999999998,"15min":0.66300000000000003},"min":{"1min":0.45800000000000002,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.014,"5min":1.464,"15min":1.464},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.71399999999999997,"15min":0.65800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.89200000000000002,"5min":1.234,"15min":1.234},"last":0.71099999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.79800000000000004,"15min":0.80300000000000005},"min":{"1min":0.442,"5min":0.437,"15min":0.437},"max":{"1min":1.0740000000000001,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.754,"15min":0.74199999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.90600000000000003,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.006}]},{"osd":4,"last update":"Mon Oct 9 06:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.89500000000000002,"15min":0.89300000000000002},"min":{"1min":0.56799999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":0.95699999999999996,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.208,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.198}]},{"osd":5,"last update":"Mon Oct 9 06:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95099999999999996},"min":{"1min":0.72399999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.2290000000000001,"5min":1.294,"15min":1.294},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.22,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.0369999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0760000000000001,"15min":1.089},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.4990000000000001,"5min":1.516,"15min":1.516},"last":1.2909999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.0249999999999999,"15min":1.0469999999999999},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.244,"5min":1.264,"15min":1.264},"last":1.0900000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.089,"15min":1.1140000000000001},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3160000000000001,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":1.238},{"interface":"front","average":{"1min":1.016,"5min":1.1879999999999999,"15min":1.2170000000000001},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.171,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.3580000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738462,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65200000000000002,"15min":0.56599999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":0.95499999999999996,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.755,"15min":0.621},"min":{"1min":0.54700000000000004,"5min":0.46300000000000002,"15min":0.38200000000000001},"max":{"1min":1.0720000000000001,"5min":1.22,"15min":1.22},"last":0.70699999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.74399999999999999,"15min":0.66700000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.871,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.627},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.72099999999999997,"15min":0.64000000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.83199999999999996,"5min":1.413,"15min":1.413},"last":0.72899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.78100000000000003,"15min":0.79900000000000004},"min":{"1min":0.379,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.022,"5min":1.387,"15min":1.387},"last":0.754},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.80400000000000005,"15min":0.79500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.163,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.873,"15min":0.84799999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.4870000000000001,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":0.879},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.423,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":0.78700000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.016,"15min":0.998},"min":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3220000000000001,"5min":1.968,"15min":1.968},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0349999999999999,"15min":1.026},"min":{"1min":0.82899999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.548,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.0840000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.071,"15min":1.121},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.4470000000000001,"5min":2.0409999999999999,"15min":2.0409999999999999},"last":1.0649999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.0329999999999999,"15min":1.0609999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.369,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.1359999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0409999999999999,"15min":1.0509999999999999},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3899999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":1.179},{"interface":"front","average":{"1min":1.103,"5min":1.079,"15min":1.075},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.4470000000000001,"5min":2.4470000000000001,"15min":2.4470000000000001},"last":1.016}]}]},{"osd":2,"up_from":18,"seq":77309411405,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.66800000000000004,"15min":0.625},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.67800000000000005,"15min":0.61799999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.0189999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.61899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.69199999999999995,"15min":0.63300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.73499999999999999,"15min":0.68700000000000006},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.0469999999999999,"5min":1.079,"15min":1.079},"last":0.66600000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.79100000000000004,"15min":0.78400000000000003},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.93799999999999994,"15min":0.92500000000000004},"min":{"1min":0.747,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.01,"15min":1.0329999999999999},"min":{"1min":0.66300000000000003,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.86499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.95899999999999996,"15min":0.98199999999999998},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.288,"5min":1.734,"15min":1.734},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.03,"5min":1,"15min":1.0009999999999999},"min":{"1min":0.73199999999999998,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.2789999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.1799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0269999999999999,"15min":1.0469999999999999},"min":{"1min":0.59599999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.046,"5min":1.0680000000000001,"15min":1.087},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.8839999999999999,"5min":1.8839999999999999,"15min":1.8839999999999999},"last":1.256}]},{"osd":7,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0409999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":2.1160000000000001,"5min":2.1160000000000001,"15min":2.1160000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0469999999999999,"15min":1.0489999999999999},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.996}]}]},{"osd":3,"up_from":25,"seq":107374182467,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.64800000000000002,"15min":0.63100000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.73999999999999999,"5min":1.232,"15min":1.232},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.73099999999999998,"15min":0.73599999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.105,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.75800000000000001,"15min":0.79000000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3640000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73499999999999999,"15min":0.78100000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88400000000000001,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.95699999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.76500000000000001,"15min":0.76900000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2130000000000001,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.79700000000000004,"15min":0.80400000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.107,"5min":1.754,"15min":1.754},"last":1.0649999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:00:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.93000000000000005,"15min":0.90300000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.347,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.361},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.96099999999999997,"15min":0.92600000000000005},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":1.0149999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.042,"15min":1.048},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.022,"5min":1.0429999999999999,"15min":1.038},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.3560000000000001,"5min":1.651,"15min":1.651},"last":1.425}]},{"osd":6,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.0629999999999999,"15min":1.1100000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.246,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.5429999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.093,"15min":1.1259999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.294}]},{"osd":7,"last update":"Mon Oct 9 06:00:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.1100000000000001,"15min":1.1279999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.421,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.492},{"interface":"front","average":{"1min":1.071,"5min":1.135,"15min":1.1459999999999999},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.665,"5min":1.665,"15min":1.665},"last":1.1930000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018939,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.83099999999999996,"15min":0.83499999999999996},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.84699999999999998,"15min":0.82299999999999995},"min":{"1min":0.57199999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.226,"5min":1.29,"15min":1.29},"last":0.996}]},{"osd":1,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.83199999999999996,"15min":0.79800000000000004},"min":{"1min":0.57099999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.079,"5min":2.3410000000000002,"15min":2.3410000000000002},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.88200000000000001,"15min":0.86199999999999999},"min":{"1min":0.57799999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.2609999999999999,"5min":1.494,"15min":1.494},"last":1.335}]},{"osd":2,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.90100000000000002,"15min":0.88100000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.379,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.2130000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.89100000000000001,"15min":0.873},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.4119999999999999,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.1180000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.88,"15min":0.85999999999999999},"min":{"1min":0.59099999999999997,"5min":0.497,"15min":0.497},"max":{"1min":1.3280000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.91400000000000003,"15min":0.87},"min":{"1min":0.60299999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.798,"5min":2.5619999999999998,"15min":2.5619999999999998},"last":0.85299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:00:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.78600000000000003,"15min":0.747},"min":{"1min":0.54700000000000004,"5min":0.372,"15min":0.372},"max":{"1min":1.2470000000000001,"5min":2.79,"15min":2.79},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.78700000000000003,"15min":0.752},"min":{"1min":0.48299999999999998,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.3240000000000001,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":1.054}]},{"osd":6,"last update":"Mon Oct 9 06:01:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.95299999999999996,"15min":0.95599999999999996},"min":{"1min":0.53100000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.266,"5min":1.891,"15min":1.891},"last":1.2849999999999999},{"interface":"front","average":{"1min":0.995,"5min":0.98499999999999999,"15min":0.97899999999999998},"min":{"1min":0.624,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.391,"5min":2.915,"15min":2.915},"last":1.1060000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0609999999999999,"15min":1.069},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3939999999999999,"5min":1.988,"15min":1.988},"last":1.377},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0609999999999999,"15min":1.071},"min":{"1min":0.59299999999999997,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.498,"5min":2.629,"15min":2.629},"last":1.3089999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822706,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.85799999999999998,"15min":0.85999999999999999},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":1.125},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.877,"15min":0.87},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":1.1439999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88900000000000001,"15min":0.88300000000000001},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.095,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.88,"15min":0.89200000000000002},"min":{"1min":0.621,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1899999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.056}]},{"osd":2,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.89000000000000001,"15min":0.89400000000000002},"min":{"1min":0.68500000000000005,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.93600000000000005,"15min":0.93600000000000005},"min":{"1min":0.70099999999999996,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.206}]},{"osd":3,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.89300000000000002,"15min":0.88100000000000001},"min":{"1min":0.68300000000000005,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.1419999999999999,"5min":1.228,"15min":1.228},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.96899999999999997,"15min":0.96699999999999997},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.7370000000000001,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":1.0820000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.80900000000000005,"15min":0.80200000000000005},"min":{"1min":0.57499999999999996,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.80600000000000005,"15min":0.77800000000000002},"min":{"1min":0.53300000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.288,"5min":1.718,"15min":1.718},"last":1.004}]},{"osd":6,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.94899999999999995,"15min":0.95299999999999996},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2909999999999999,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.61899999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.3340000000000001,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":0.85499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:00:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.071,"15min":1.081},"min":{"1min":0.57199999999999995,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.575,"5min":1.718,"15min":1.718},"last":1.097},{"interface":"front","average":{"1min":1.014,"5min":1.1020000000000001,"15min":1.1160000000000001},"min":{"1min":0.59499999999999997,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.244,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.0289999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:01:27.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:01:28.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:28 smithi125 bash[27513]: audit 2023-10-09T06:01:27.152866+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:28.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:28 smithi125 bash[33919]: audit 2023-10-09T06:01:27.152866+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:28.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:28] "GET /metrics HTTP/1.1" 200 155659 "" "Prometheus/2.33.4" 2023-10-09T06:01:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:28 smithi155 bash[27817]: audit 2023-10-09T06:01:27.152866+0000 mgr.y (mgr.14430) 89 : audit [DBG] from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:29.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:29 smithi125 bash[27513]: cluster 2023-10-09T06:01:28.212302+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:29 smithi125 bash[33919]: cluster 2023-10-09T06:01:28.212302+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:29 smithi155 bash[27817]: cluster 2023-10-09T06:01:28.212302+0000 mgr.y (mgr.14430) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:31.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:31 smithi155 bash[27817]: cluster 2023-10-09T06:01:30.212869+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:31.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:31 smithi125 bash[27513]: cluster 2023-10-09T06:01:30.212869+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:31.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:31 smithi125 bash[33919]: cluster 2023-10-09T06:01:30.212869+0000 mgr.y (mgr.14430) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:32.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:01:31 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:01:32.682 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:33.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:33 smithi125 bash[27513]: cluster 2023-10-09T06:01:32.213498+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:33.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:33 smithi125 bash[33919]: cluster 2023-10-09T06:01:32.213498+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:33 smithi155 bash[27817]: cluster 2023-10-09T06:01:32.213498+0000 mgr.y (mgr.14430) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:35.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:35 smithi125 bash[27513]: cluster 2023-10-09T06:01:34.213830+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:35 smithi125 bash[33919]: cluster 2023-10-09T06:01:34.213830+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:35.749 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:01:35.753 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:01:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:35 smithi155 bash[27817]: cluster 2023-10-09T06:01:34.213830+0000 mgr.y (mgr.14430) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:36.582 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2023-10-09T06:01:34.213692+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380972,"kb_used_data":4204,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580244,"statfs":{"total":767960285184,"available":765522169856,"internally_reserved":0,"allocated":4304896,"data_stored":1762545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.462366+0000","last_change":"2023-10-09T05:59:52.469538+0000","last_active":"2023-10-09T05:59:53.462366+0000","last_peered":"2023-10-09T05:59:53.462366+0000","last_clean":"2023-10-09T05:59:53.462366+0000","last_became_active":"2023-10-09T05:59:52.469011+0000","last_became_peered":"2023-10-09T05:59:52.469011+0000","last_unstale":"2023-10-09T05:59:53.462366+0000","last_undegraded":"2023-10-09T05:59:53.462366+0000","last_fullsized":"2023-10-09T05:59:53.462366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461573+0000","last_change":"2023-10-09T05:59:52.779228+0000","last_active":"2023-10-09T05:59:53.461573+0000","last_peered":"2023-10-09T05:59:53.461573+0000","last_clean":"2023-10-09T05:59:53.461573+0000","last_became_active":"2023-10-09T05:59:52.777994+0000","last_became_peered":"2023-10-09T05:59:52.777994+0000","last_unstale":"2023-10-09T05:59:53.461573+0000","last_undegraded":"2023-10-09T05:59:53.461573+0000","last_fullsized":"2023-10-09T05:59:53.461573+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630679+0000","last_change":"2023-10-09T05:59:52.778611+0000","last_active":"2023-10-09T05:59:54.630679+0000","last_peered":"2023-10-09T05:59:54.630679+0000","last_clean":"2023-10-09T05:59:54.630679+0000","last_became_active":"2023-10-09T05:59:52.777893+0000","last_became_peered":"2023-10-09T05:59:52.777893+0000","last_unstale":"2023-10-09T05:59:54.630679+0000","last_undegraded":"2023-10-09T05:59:54.630679+0000","last_fullsized":"2023-10-09T05:59:54.630679+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630490+0000","last_change":"2023-10-09T05:59:52.583913+0000","last_active":"2023-10-09T05:59:54.630490+0000","last_peered":"2023-10-09T05:59:54.630490+0000","last_clean":"2023-10-09T05:59:54.630490+0000","last_became_active":"2023-10-09T05:59:52.583298+0000","last_became_peered":"2023-10-09T05:59:52.583298+0000","last_unstale":"2023-10-09T05:59:54.630490+0000","last_undegraded":"2023-10-09T05:59:54.630490+0000","last_fullsized":"2023-10-09T05:59:54.630490+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.630469+0000","last_change":"2023-10-09T05:58:52.535269+0000","last_active":"2023-10-09T05:59:54.630469+0000","last_peered":"2023-10-09T05:59:54.630469+0000","last_clean":"2023-10-09T05:59:54.630469+0000","last_became_active":"2023-10-09T05:58:52.534584+0000","last_became_peered":"2023-10-09T05:58:52.534584+0000","last_unstale":"2023-10-09T05:59:54.630469+0000","last_undegraded":"2023-10-09T05:59:54.630469+0000","last_fullsized":"2023-10-09T05:59:54.630469+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.471806+0000","last_change":"2023-10-09T05:59:52.469599+0000","last_active":"2023-10-09T05:59:54.471806+0000","last_peered":"2023-10-09T05:59:54.471806+0000","last_clean":"2023-10-09T05:59:54.471806+0000","last_became_active":"2023-10-09T05:59:52.469155+0000","last_became_peered":"2023-10-09T05:59:52.469155+0000","last_unstale":"2023-10-09T05:59:54.471806+0000","last_undegraded":"2023-10-09T05:59:54.471806+0000","last_fullsized":"2023-10-09T05:59:54.471806+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461109+0000","last_change":"2023-10-09T05:59:52.778084+0000","last_active":"2023-10-09T05:59:53.461109+0000","last_peered":"2023-10-09T05:59:53.461109+0000","last_clean":"2023-10-09T05:59:53.461109+0000","last_became_active":"2023-10-09T05:59:52.777374+0000","last_became_peered":"2023-10-09T05:59:52.777374+0000","last_unstale":"2023-10-09T05:59:53.461109+0000","last_undegraded":"2023-10-09T05:59:53.461109+0000","last_fullsized":"2023-10-09T05:59:53.461109+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:54.471782+0000","last_change":"2023-10-09T05:59:52.467577+0000","last_active":"2023-10-09T05:59:54.471782+0000","last_peered":"2023-10-09T05:59:54.471782+0000","last_clean":"2023-10-09T05:59:54.471782+0000","last_became_active":"2023-10-09T05:59:52.467143+0000","last_became_peered":"2023-10-09T05:59:52.467143+0000","last_unstale":"2023-10-09T05:59:54.471782+0000","last_undegraded":"2023-10-09T05:59:54.471782+0000","last_fullsized":"2023-10-09T05:59:54.471782+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-09T05:59:53.461366+0000","last_change":"2023-10-09T05:59:52.583128+0000","last_active":"2023-10-09T05:59:53.461366+0000","last_peered":"2023-10-09T05:59:53.461366+0000","last_clean":"2023-10-09T05:59:53.461366+0000","last_became_active":"2023-10-09T05:59:52.582692+0000","last_became_peered":"2023-10-09T05:59:52.582692+0000","last_unstale":"2023-10-09T05:59:53.461366+0000","last_undegraded":"2023-10-09T05:59:53.461366+0000","last_fullsized":"2023-10-09T05:59:53.461366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430242,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.86699999999999999,"15min":0.88100000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.0720000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.4610000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.92900000000000005,"15min":0.93400000000000005},"min":{"1min":0.61399999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1859999999999999,"5min":1.452,"15min":1.452},"last":1.032}]},{"osd":1,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.879,"15min":0.88900000000000001},"min":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.999},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.96899999999999997,"15min":0.98199999999999998},"min":{"1min":0.68300000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.125,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.3080000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.89000000000000001,"15min":0.90100000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.1579999999999999,"5min":1.173,"15min":1.173},"last":1.165},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.90300000000000002,"15min":0.90200000000000002},"min":{"1min":0.66100000000000003,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.3440000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.2849999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.95499999999999996,"15min":0.96499999999999997},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.2789999999999999,"5min":1.335,"15min":1.335},"last":1.2589999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.95399999999999996,"15min":0.95599999999999996},"min":{"1min":0.64500000000000002,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.274,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.093}]},{"osd":4,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.94799999999999995,"15min":0.95799999999999996},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.2470000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.127},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.89400000000000002,"15min":0.89900000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2230000000000001,"5min":1.51,"15min":1.51},"last":1.0649999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.96799999999999997,"15min":0.98199999999999998},"min":{"1min":0.57899999999999996,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1659999999999999,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.999,"15min":1.0109999999999999},"min":{"1min":0.64800000000000002,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2729999999999999,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.23}]},{"osd":6,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.0389999999999999,"15min":1.048},"min":{"1min":0.66400000000000003,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3240000000000001,"5min":1.508,"15min":1.508},"last":1.397},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.036,"15min":1.0409999999999999},"min":{"1min":0.69799999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3340000000000001,"5min":2.0019999999999998,"15min":2.0019999999999998},"last":1.3520000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626474,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.86499999999999999,"15min":0.88100000000000001},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.1799999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86399999999999999,"15min":0.88600000000000001},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.94499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.81399999999999995,"15min":0.82499999999999996},"min":{"1min":0.51900000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.0469999999999999,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.88},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.83199999999999996,"15min":0.81799999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0649999999999999,"5min":1.278,"15min":1.278},"last":0.89900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.139,"5min":1.216,"15min":1.216},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.93700000000000006,"15min":0.96499999999999997},"min":{"1min":0.66600000000000004,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2709999999999999,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":1.056}]},{"osd":3,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.86299999999999999,"15min":0.86099999999999999},"min":{"1min":0.622,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.105,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.95099999999999996,"15min":0.94299999999999995},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.252,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.1060000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.872,"15min":0.874},"min":{"1min":0.51100000000000001,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.21,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.88900000000000001,"15min":0.88},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.222,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.1339999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:01:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.878,"15min":0.84399999999999997},"min":{"1min":0.56100000000000005,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.3799999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.282},{"interface":"front","average":{"1min":1,"5min":0.93500000000000005,"15min":0.91100000000000003},"min":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.089}]},{"osd":7,"last update":"Mon Oct 9 06:00:58 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.018,"15min":1.016},"min":{"1min":0.47399999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3480000000000001,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.3320000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.034,"15min":1.034},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.3819999999999999,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.1970000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574935,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.69899999999999995,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.36799999999999999},"max":{"1min":0.93799999999999994,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.3600000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.68300000000000005,"15min":0.60699999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":0.93100000000000005,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.48299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73299999999999998,"15min":0.66300000000000003},"min":{"1min":0.45800000000000002,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.014,"5min":1.464,"15min":1.464},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.71399999999999997,"15min":0.65800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.89200000000000002,"5min":1.234,"15min":1.234},"last":0.84599999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.79800000000000004,"15min":0.80300000000000005},"min":{"1min":0.442,"5min":0.437,"15min":0.437},"max":{"1min":1.0740000000000001,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.754,"15min":0.74199999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.90600000000000003,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.95299999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.89500000000000002,"15min":0.89300000000000002},"min":{"1min":0.56799999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":0.95699999999999996,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.97999999999999998,"15min":0.97999999999999998},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.208,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.149}]},{"osd":5,"last update":"Mon Oct 9 06:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95099999999999996},"min":{"1min":0.72399999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.2290000000000001,"5min":1.294,"15min":1.294},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.96699999999999997,"15min":0.96099999999999997},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.22,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.1919999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0760000000000001,"15min":1.089},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.4990000000000001,"5min":1.516,"15min":1.516},"last":1.2490000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.0249999999999999,"15min":1.0469999999999999},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.244,"5min":1.264,"15min":1.264},"last":1.024}]},{"osd":7,"last update":"Mon Oct 9 06:00:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.089,"15min":1.1140000000000001},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3160000000000001,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.016,"5min":1.1879999999999999,"15min":1.2170000000000001},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.171,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.3100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738463,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:00:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65200000000000002,"15min":0.56599999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":0.95499999999999996,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.755,"15min":0.621},"min":{"1min":0.54700000000000004,"5min":0.46300000000000002,"15min":0.38200000000000001},"max":{"1min":1.0720000000000001,"5min":1.22,"15min":1.22},"last":0.66600000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:01:24 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.74399999999999999,"15min":0.66700000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.871,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.72099999999999997,"15min":0.64000000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.83199999999999996,"5min":1.413,"15min":1.413},"last":0.751}]},{"osd":3,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.78100000000000003,"15min":0.79900000000000004},"min":{"1min":0.379,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.022,"5min":1.387,"15min":1.387},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.80400000000000005,"15min":0.79500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.163,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.873}]},{"osd":4,"last update":"Mon Oct 9 06:00:46 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.873,"15min":0.84799999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.4870000000000001,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.997,"15min":0.99299999999999999},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.423,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":0.93700000000000006}]},{"osd":5,"last update":"Mon Oct 9 06:00:29 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.016,"15min":0.998},"min":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3220000000000001,"5min":1.968,"15min":1.968},"last":0.97899999999999998},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0349999999999999,"15min":1.026},"min":{"1min":0.82899999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.548,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.175}]},{"osd":6,"last update":"Mon Oct 9 06:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.071,"15min":1.121},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.4470000000000001,"5min":2.0409999999999999,"15min":2.0409999999999999},"last":1.034},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.0329999999999999,"15min":1.0609999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.369,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.222}]},{"osd":7,"last update":"Mon Oct 9 06:00:51 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0409999999999999,"15min":1.0509999999999999},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3899999999999999,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.079,"15min":1.075},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.4470000000000001,"5min":2.4470000000000001,"15min":2.4470000000000001},"last":0.98899999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411406,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.66800000000000004,"15min":0.625},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.67800000000000005,"15min":0.61799999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.0189999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.61899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.69199999999999995,"15min":0.63300000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.73499999999999999,"15min":0.68700000000000006},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.0469999999999999,"5min":1.079,"15min":1.079},"last":0.66600000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.76800000000000002,"15min":0.76400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.79100000000000004,"15min":0.78400000000000003},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":0.76700000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:00:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.93799999999999994,"15min":0.92500000000000004},"min":{"1min":0.747,"5min":0.68600000000000005,"15min":0.68600000000000005},"max":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.01,"15min":1.0329999999999999},"min":{"1min":0.66300000000000003,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.86499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.95899999999999996,"15min":0.98199999999999998},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.288,"5min":1.734,"15min":1.734},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.03,"5min":1,"15min":1.0009999999999999},"min":{"1min":0.73199999999999998,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.2789999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.1799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0269999999999999,"15min":1.0469999999999999},"min":{"1min":0.59599999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.046,"5min":1.0680000000000001,"15min":1.087},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.8839999999999999,"5min":1.8839999999999999,"15min":1.8839999999999999},"last":1.256}]},{"osd":7,"last update":"Mon Oct 9 06:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0409999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":2.1160000000000001,"5min":2.1160000000000001,"15min":2.1160000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0469999999999999,"15min":1.0489999999999999},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.776,"5min":1.776,"15min":1.776},"last":0.996}]}]},{"osd":3,"up_from":25,"seq":107374182468,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.64800000000000002,"15min":0.63100000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.73999999999999999,"5min":1.232,"15min":1.232},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.73099999999999998,"15min":0.73599999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.105,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.86299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.75800000000000001,"15min":0.79000000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3640000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.73499999999999999,"15min":0.78100000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88400000000000001,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.627}]},{"osd":2,"last update":"Mon Oct 9 06:01:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.76500000000000001,"15min":0.76900000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.2130000000000001,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.79700000000000004,"15min":0.80400000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.107,"5min":1.754,"15min":1.754},"last":0.92200000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:00:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.93000000000000005,"15min":0.90300000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.347,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.96099999999999997,"15min":0.92600000000000005},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":1.0009999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:00:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.042,"15min":1.048},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.165},{"interface":"front","average":{"1min":1.022,"5min":1.0429999999999999,"15min":1.038},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.3560000000000001,"5min":1.651,"15min":1.651},"last":1.1439999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.0629999999999999,"15min":1.1100000000000001},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.246,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.224},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.093,"15min":1.1259999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":1.339}]},{"osd":7,"last update":"Mon Oct 9 06:00:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.1100000000000001,"15min":1.1279999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.421,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.353},{"interface":"front","average":{"1min":1.071,"5min":1.135,"15min":1.1459999999999999},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.665,"5min":1.665,"15min":1.665},"last":1.286}]}]},{"osd":4,"up_from":30,"seq":128849018940,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.83099999999999996,"15min":0.83499999999999996},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.84699999999999998,"15min":0.82299999999999995},"min":{"1min":0.57199999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.226,"5min":1.29,"15min":1.29},"last":0.79700000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.83199999999999996,"15min":0.79800000000000004},"min":{"1min":0.57099999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.079,"5min":2.3410000000000002,"15min":2.3410000000000002},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.88200000000000001,"15min":0.86199999999999999},"min":{"1min":0.57799999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.2609999999999999,"5min":1.494,"15min":1.494},"last":1.2270000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.90100000000000002,"15min":0.88100000000000001},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.379,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.89100000000000001,"15min":0.873},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.4119999999999999,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.88}]},{"osd":3,"last update":"Mon Oct 9 06:00:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.88,"15min":0.85999999999999999},"min":{"1min":0.59099999999999997,"5min":0.497,"15min":0.497},"max":{"1min":1.3280000000000001,"5min":1.3600000000000001,"15min":1.3600000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.91400000000000003,"15min":0.87},"min":{"1min":0.60299999999999998,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.798,"5min":2.5619999999999998,"15min":2.5619999999999998},"last":1.0900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:01:27 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.82999999999999996,"15min":0.76200000000000001},"min":{"1min":0.54600000000000004,"5min":0.372,"15min":0.372},"max":{"1min":1.198,"5min":2.79,"15min":2.79},"last":1.198},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.83899999999999997,"15min":0.76900000000000002},"min":{"1min":0.57799999999999996,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.373,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":1.0109999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:01:10 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.95299999999999996,"15min":0.95599999999999996},"min":{"1min":0.53100000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.266,"5min":1.891,"15min":1.891},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.995,"5min":0.98499999999999999,"15min":0.97899999999999998},"min":{"1min":0.624,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.391,"5min":2.915,"15min":2.915},"last":1.2569999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:00:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0609999999999999,"15min":1.069},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3939999999999999,"5min":1.988,"15min":1.988},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0609999999999999,"15min":1.071},"min":{"1min":0.59299999999999997,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.498,"5min":2.629,"15min":2.629},"last":1.169}]}]},{"osd":5,"up_from":36,"seq":154618822707,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.85799999999999998,"15min":0.85999999999999999},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.877,"15min":0.87},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.83999999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.88900000000000001,"15min":0.88300000000000001},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.095,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.88,"15min":0.89200000000000002},"min":{"1min":0.621,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1899999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.748}]},{"osd":2,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.89000000000000001,"15min":0.89400000000000002},"min":{"1min":0.68500000000000005,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.874},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.93600000000000005,"15min":0.93600000000000005},"min":{"1min":0.70099999999999996,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.94899999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.89300000000000002,"15min":0.88100000000000001},"min":{"1min":0.68300000000000005,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.1419999999999999,"5min":1.228,"15min":1.228},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.96899999999999997,"15min":0.96699999999999997},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.7370000000000001,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":0.90800000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.80900000000000005,"15min":0.80200000000000005},"min":{"1min":0.57499999999999996,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.80600000000000005,"15min":0.77800000000000002},"min":{"1min":0.53300000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.288,"5min":1.718,"15min":1.718},"last":0.77400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:01:14 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.94899999999999995,"15min":0.95299999999999996},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2909999999999999,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.008},{"interface":"front","average":{"1min":1.026,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.61899999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.3340000000000001,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.1020000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:00:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.071,"15min":1.081},"min":{"1min":0.57199999999999995,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.575,"5min":1.718,"15min":1.718},"last":1.056},{"interface":"front","average":{"1min":1.014,"5min":1.1020000000000001,"15min":1.1160000000000001},"min":{"1min":0.59499999999999997,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.244,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.026}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:01:36.584 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:01:36.585 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T06:01:36.585 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T06:01:36.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T06:01:37.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:37 smithi125 bash[27513]: audit 2023-10-09T06:01:35.740733+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:37 smithi125 bash[27513]: cluster 2023-10-09T06:01:36.214390+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:37 smithi125 bash[33919]: audit 2023-10-09T06:01:35.740733+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:37 smithi125 bash[33919]: cluster 2023-10-09T06:01:36.214390+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:37 smithi155 bash[27817]: audit 2023-10-09T06:01:35.740733+0000 mgr.y (mgr.14430) 94 : audit [DBG] from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:01:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:37 smithi155 bash[27817]: cluster 2023-10-09T06:01:36.214390+0000 mgr.y (mgr.14430) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:39.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:38 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:38] "GET /metrics HTTP/1.1" 200 155659 "" "Prometheus/2.33.4" 2023-10-09T06:01:39.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:39 smithi125 bash[27513]: cluster 2023-10-09T06:01:38.214920+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:39 smithi125 bash[33919]: cluster 2023-10-09T06:01:38.214920+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:39 smithi155 bash[27817]: cluster 2023-10-09T06:01:38.214920+0000 mgr.y (mgr.14430) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:41.355 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:01:41.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:41 smithi155 bash[27817]: cluster 2023-10-09T06:01:40.215450+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:41 smithi125 bash[27513]: cluster 2023-10-09T06:01:40.215450+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:41 smithi125 bash[33919]: cluster 2023-10-09T06:01:40.215450+0000 mgr.y (mgr.14430) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:42.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:01:41 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:01:43.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:43 smithi125 bash[27513]: cluster 2023-10-09T06:01:42.216120+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:43.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:43 smithi125 bash[33919]: cluster 2023-10-09T06:01:42.216120+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:43 smithi155 bash[27817]: cluster 2023-10-09T06:01:42.216120+0000 mgr.y (mgr.14430) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:44.636 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:01:45.421 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T06:01:45.421 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T06:01:45.421 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T06:01:45.422 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T06:01:45.430 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T06:01:45.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T06:01:45.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:45 smithi125 bash[27513]: cluster 2023-10-09T06:01:44.216524+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:45 smithi125 bash[27513]: audit 2023-10-09T06:01:44.634485+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.125:0/338524859' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:01:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:45 smithi125 bash[33919]: cluster 2023-10-09T06:01:44.216524+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:45 smithi125 bash[33919]: audit 2023-10-09T06:01:44.634485+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.125:0/338524859' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:01:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:45 smithi155 bash[27817]: cluster 2023-10-09T06:01:44.216524+0000 mgr.y (mgr.14430) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:45 smithi155 bash[27817]: audit 2023-10-09T06:01:44.634485+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.125:0/338524859' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:01:47.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:47 smithi125 bash[27513]: cluster 2023-10-09T06:01:46.217019+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:47 smithi125 bash[33919]: cluster 2023-10-09T06:01:46.217019+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:47 smithi155 bash[27817]: cluster 2023-10-09T06:01:46.217019+0000 mgr.y (mgr.14430) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:49.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:48] "GET /metrics HTTP/1.1" 200 155677 "" "Prometheus/2.33.4" 2023-10-09T06:01:49.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[27513]: cluster 2023-10-09T06:01:48.217528+0000 mgr.y (mgr.14430) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:49.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[27513]: audit 2023-10-09T06:01:48.799727+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:49.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[27513]: audit 2023-10-09T06:01:48.800572+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:49.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[33919]: cluster 2023-10-09T06:01:48.217528+0000 mgr.y (mgr.14430) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:49.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[33919]: audit 2023-10-09T06:01:48.799727+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:49.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:49 smithi125 bash[33919]: audit 2023-10-09T06:01:48.800572+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:49 smithi155 bash[27817]: cluster 2023-10-09T06:01:48.217528+0000 mgr.y (mgr.14430) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:49 smithi155 bash[27817]: audit 2023-10-09T06:01:48.799727+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:49 smithi155 bash[27817]: audit 2023-10-09T06:01:48.800572+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T06:01:50.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[33919]: audit 2023-10-09T06:01:48.798655+0000 mgr.y (mgr.14430) 102 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[33919]: audit 2023-10-09T06:01:49.624252+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[33919]: cluster 2023-10-09T06:01:49.624306+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[33919]: audit 2023-10-09T06:01:49.627455+0000 mon.c (mon.2) 68 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[33919]: audit 2023-10-09T06:01:49.628020+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: audit 2023-10-09T06:01:48.798655+0000 mgr.y (mgr.14430) 102 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: audit 2023-10-09T06:01:49.624252+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: cluster 2023-10-09T06:01:49.624306+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T06:01:50.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: audit 2023-10-09T06:01:49.627455+0000 mon.c (mon.2) 68 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: audit 2023-10-09T06:01:49.628020+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:50 smithi125 bash[27513]: debug 2023-10-09T06:01:50.649+0000 7f79d6d93700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T06:01:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:50 smithi155 bash[27817]: audit 2023-10-09T06:01:48.798655+0000 mgr.y (mgr.14430) 102 : audit [DBG] from='client.14625 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:01:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:50 smithi155 bash[27817]: audit 2023-10-09T06:01:49.624252+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T06:01:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:50 smithi155 bash[27817]: cluster 2023-10-09T06:01:49.624306+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-09T06:01:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:50 smithi155 bash[27817]: audit 2023-10-09T06:01:49.627455+0000 mon.c (mon.2) 68 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:50 smithi155 bash[27817]: audit 2023-10-09T06:01:49.628020+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.218119+0000 mgr.y (mgr.14430) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.634576+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.634776+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.647130+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.651280+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.652477+0000 mon.a (mon.0) 633 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.652981+0000 mon.a (mon.0) 634 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T06:01:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.663696+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.663955+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cluster 2023-10-09T06:01:50.664054+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:0 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cephadm 2023-10-09T06:01:50.671798+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.678888+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.687501+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:50.919005+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.190539+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.191911+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.200237+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.210864+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.212484+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.212864+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.217112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: audit 2023-10-09T06:01:51.218274+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[33919]: cephadm 2023-10-09T06:01:51.219245+0000 mgr.y (mgr.14430) 105 : cephadm [INF] Deploying daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.218119+0000 mgr.y (mgr.14430) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.634576+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.634776+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.647130+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.651280+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.652477+0000 mon.a (mon.0) 633 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.652981+0000 mon.a (mon.0) 634 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.663696+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.663955+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cluster 2023-10-09T06:01:50.664054+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:0 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cephadm 2023-10-09T06:01:50.671798+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.678888+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.687501+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:50.919005+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.190539+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.191911+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.200237+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.210864+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.212484+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.212864+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.217112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: audit 2023-10-09T06:01:51.218274+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:51 smithi125 bash[27513]: cephadm 2023-10-09T06:01:51.219245+0000 mgr.y (mgr.14430) 105 : cephadm [INF] Deploying daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:01:51.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:01:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.218119+0000 mgr.y (mgr.14430) 103 : cluster [DBG] pgmap v81: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.634576+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.634776+0000 mon.a (mon.0) 631 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.647130+0000 mon.c (mon.2) 69 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.651280+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.652477+0000 mon.a (mon.0) 633 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.652981+0000 mon.a (mon.0) 634 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.663696+0000 mon.a (mon.0) 635 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.663955+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cluster 2023-10-09T06:01:50.664054+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:0 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cephadm 2023-10-09T06:01:50.671798+0000 mgr.y (mgr.14430) 104 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.678888+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.687501+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:50.919005+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.190539+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.191911+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.200237+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.210864+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.212484+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.212864+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.217112+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: audit 2023-10-09T06:01:51.218274+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:51 smithi155 bash[27817]: cephadm 2023-10-09T06:01:51.219245+0000 mgr.y (mgr.14430) 105 : cephadm [INF] Deploying daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:01:53.023 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:52 smithi125 bash[27513]: cluster 2023-10-09T06:01:51.665222+0000 mon.a (mon.0) 644 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T06:01:53.023 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:52 smithi125 bash[27513]: cluster 2023-10-09T06:01:52.218796+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:53.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:52 smithi125 bash[33919]: cluster 2023-10-09T06:01:51.665222+0000 mon.a (mon.0) 644 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T06:01:53.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:52 smithi125 bash[33919]: cluster 2023-10-09T06:01:52.218796+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:52 smithi155 bash[27817]: cluster 2023-10-09T06:01:51.665222+0000 mon.a (mon.0) 644 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-09T06:01:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:52 smithi155 bash[27817]: cluster 2023-10-09T06:01:52.218796+0000 mgr.y (mgr.14430) 106 : cluster [DBG] pgmap v85: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:53.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:53 smithi125 bash[27513]: cluster 2023-10-09T06:01:52.674545+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T06:01:53.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:53 smithi125 bash[33919]: cluster 2023-10-09T06:01:52.674545+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T06:01:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:53 smithi155 bash[27817]: cluster 2023-10-09T06:01:52.674545+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-09T06:01:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: cluster 2023-10-09T06:01:54.219216+0000 mgr.y (mgr.14430) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.425359+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.919773+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.922720+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.923185+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.928846+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:55 smithi155 bash[27817]: audit 2023-10-09T06:01:54.929990+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: cluster 2023-10-09T06:01:54.219216+0000 mgr.y (mgr.14430) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.425359+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.919773+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.922720+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.923185+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.928846+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[33919]: audit 2023-10-09T06:01:54.929990+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: cluster 2023-10-09T06:01:54.219216+0000 mgr.y (mgr.14430) 107 : cluster [DBG] pgmap v87: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.425359+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.919773+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.922720+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.923185+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.928846+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T06:01:55.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:55 smithi125 bash[27513]: audit 2023-10-09T06:01:54.929990+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:56.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:56 smithi125 bash[27513]: cephadm 2023-10-09T06:01:54.930927+0000 mgr.y (mgr.14430) 108 : cephadm [INF] Deploying daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:01:56.626 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:56 smithi125 bash[33919]: cephadm 2023-10-09T06:01:54.930927+0000 mgr.y (mgr.14430) 108 : cephadm [INF] Deploying daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:01:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:56 smithi155 bash[27817]: cephadm 2023-10-09T06:01:54.930927+0000 mgr.y (mgr.14430) 108 : cephadm [INF] Deploying daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:01:57.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T06:01:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:57 smithi155 bash[27817]: cluster 2023-10-09T06:01:56.220157+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:57.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:57 smithi125 bash[27513]: cluster 2023-10-09T06:01:56.220157+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:57.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:57 smithi125 bash[33919]: cluster 2023-10-09T06:01:56.220157+0000 mgr.y (mgr.14430) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:58.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.441768+0000 mon.a (mon.0) 650 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:boot 2023-10-09T06:01:58.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.441818+0000 mon.a (mon.0) 651 : cluster [INF] daemon mds.foo.smithi125.yetyhy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T06:01:58.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.441875+0000 mon.a (mon.0) 652 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T06:01:58.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.441914+0000 mon.a (mon.0) 653 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.441941+0000 mon.a (mon.0) 654 : cluster [INF] Cluster is now healthy 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: audit 2023-10-09T06:01:57.442215+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.442236+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.447637+0000 mon.a (mon.0) 656 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:creating} 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: cluster 2023-10-09T06:01:57.469460+0000 mon.a (mon.0) 657 : cluster [INF] daemon mds.foo.smithi125.yetyhy is now active in filesystem foo as rank 0 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: audit 2023-10-09T06:01:57.666629+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:58.719 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:58 smithi155 bash[27817]: audit 2023-10-09T06:01:57.671749+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.441768+0000 mon.a (mon.0) 650 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:boot 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.441818+0000 mon.a (mon.0) 651 : cluster [INF] daemon mds.foo.smithi125.yetyhy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.441875+0000 mon.a (mon.0) 652 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.441914+0000 mon.a (mon.0) 653 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.441941+0000 mon.a (mon.0) 654 : cluster [INF] Cluster is now healthy 2023-10-09T06:01:58.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: audit 2023-10-09T06:01:57.442215+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.442236+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.447637+0000 mon.a (mon.0) 656 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:creating} 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: cluster 2023-10-09T06:01:57.469460+0000 mon.a (mon.0) 657 : cluster [INF] daemon mds.foo.smithi125.yetyhy is now active in filesystem foo as rank 0 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: audit 2023-10-09T06:01:57.666629+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27513]: audit 2023-10-09T06:01:57.671749+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.441768+0000 mon.a (mon.0) 650 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:boot 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.441818+0000 mon.a (mon.0) 651 : cluster [INF] daemon mds.foo.smithi125.yetyhy assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.441875+0000 mon.a (mon.0) 652 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T06:01:58.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.441914+0000 mon.a (mon.0) 653 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.441941+0000 mon.a (mon.0) 654 : cluster [INF] Cluster is now healthy 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: audit 2023-10-09T06:01:57.442215+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.442236+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.447637+0000 mon.a (mon.0) 656 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:creating} 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: cluster 2023-10-09T06:01:57.469460+0000 mon.a (mon.0) 657 : cluster [INF] daemon mds.foo.smithi125.yetyhy is now active in filesystem foo as rank 0 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: audit 2023-10-09T06:01:57.666629+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[33919]: audit 2023-10-09T06:01:57.671749+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:58.751 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:01:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:01:58] "GET /metrics HTTP/1.1" 200 160717 "" "Prometheus/2.33.4" 2023-10-09T06:01:59.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[33919]: cluster 2023-10-09T06:01:58.220819+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:59.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[33919]: cluster 2023-10-09T06:01:58.456450+0000 mon.a (mon.0) 659 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:active 2023-10-09T06:01:59.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[33919]: cluster 2023-10-09T06:01:58.456507+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 2023-10-09T06:01:59.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[27513]: cluster 2023-10-09T06:01:58.220819+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:59.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[27513]: cluster 2023-10-09T06:01:58.456450+0000 mon.a (mon.0) 659 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:active 2023-10-09T06:01:59.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:01:59 smithi125 bash[27513]: cluster 2023-10-09T06:01:58.456507+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 2023-10-09T06:01:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:59 smithi155 bash[27817]: cluster 2023-10-09T06:01:58.220819+0000 mgr.y (mgr.14430) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:01:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:59 smithi155 bash[27817]: cluster 2023-10-09T06:01:58.456450+0000 mon.a (mon.0) 659 : cluster [DBG] mds.? [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] up:active 2023-10-09T06:01:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:01:59 smithi155 bash[27817]: cluster 2023-10-09T06:01:58.456507+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 2023-10-09T06:02:00.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[27513]: audit 2023-10-09T06:01:59.536503+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[27513]: audit 2023-10-09T06:01:59.673589+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[27513]: cluster 2023-10-09T06:01:59.673705+0000 mon.a (mon.0) 662 : cluster [DBG] mds.? [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] up:boot 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[27513]: cluster 2023-10-09T06:01:59.674344+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[27513]: cluster 2023-10-09T06:01:59.677529+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[33919]: audit 2023-10-09T06:01:59.536503+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[33919]: audit 2023-10-09T06:01:59.673589+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[33919]: cluster 2023-10-09T06:01:59.673705+0000 mon.a (mon.0) 662 : cluster [DBG] mds.? [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] up:boot 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[33919]: cluster 2023-10-09T06:01:59.674344+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:00 smithi125 bash[33919]: cluster 2023-10-09T06:01:59.677529+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:00 smithi155 bash[27817]: audit 2023-10-09T06:01:59.536503+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:00 smithi155 bash[27817]: audit 2023-10-09T06:01:59.673589+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:02:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:00 smithi155 bash[27817]: cluster 2023-10-09T06:01:59.673705+0000 mon.a (mon.0) 662 : cluster [DBG] mds.? [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] up:boot 2023-10-09T06:02:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:00 smithi155 bash[27817]: cluster 2023-10-09T06:01:59.674344+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:00 smithi155 bash[27817]: cluster 2023-10-09T06:01:59.677529+0000 mon.a (mon.0) 664 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:02:01.648 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T06:02:01.657 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T06:02:01.657 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T06:02:01.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T06:02:01.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:01 smithi155 bash[27817]: cluster 2023-10-09T06:02:00.221463+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:01 smithi125 bash[27513]: cluster 2023-10-09T06:02:00.221463+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:01.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:01 smithi125 bash[33919]: cluster 2023-10-09T06:02:00.221463+0000 mgr.y (mgr.14430) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:01 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:03.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:03 smithi125 bash[27513]: cluster 2023-10-09T06:02:02.222685+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:03.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:03 smithi125 bash[33919]: cluster 2023-10-09T06:02:02.222685+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:03 smithi155 bash[27817]: cluster 2023-10-09T06:02:02.222685+0000 mgr.y (mgr.14430) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:03.583259+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.126962+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.129854+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.131355+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.138114+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.146708+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.148880+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.150078+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[27513]: audit 2023-10-09T06:02:04.156696+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:03.583259+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.126962+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.129854+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.131355+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.138114+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.146708+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.148880+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.150078+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:04 smithi125 bash[33919]: audit 2023-10-09T06:02:04.156696+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:03.583259+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.126962+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.129854+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.131355+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.138114+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.146708+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.148880+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.150078+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:04 smithi155 bash[27817]: audit 2023-10-09T06:02:04.156696+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:05 smithi125 bash[27513]: cluster 2023-10-09T06:02:04.223315+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:05 smithi125 bash[33919]: cluster 2023-10-09T06:02:04.223315+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:05 smithi155 bash[27817]: cluster 2023-10-09T06:02:04.223315+0000 mgr.y (mgr.14430) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:06.199 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:07.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:07 smithi125 bash[33919]: cluster 2023-10-09T06:02:06.224199+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:07 smithi125 bash[27513]: cluster 2023-10-09T06:02:06.224199+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:07 smithi155 bash[27817]: cluster 2023-10-09T06:02:06.224199+0000 mgr.y (mgr.14430) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:09.040 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:08] "GET /metrics HTTP/1.1" 200 160717 "" "Prometheus/2.33.4" 2023-10-09T06:02:09.490 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:02:09.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[27513]: cluster 2023-10-09T06:02:08.224764+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[27513]: audit 2023-10-09T06:02:09.426922+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[27513]: audit 2023-10-09T06:02:09.479532+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.125:0/83317155' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[27513]: audit 2023-10-09T06:02:09.561546+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[33919]: cluster 2023-10-09T06:02:08.224764+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[33919]: audit 2023-10-09T06:02:09.426922+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[33919]: audit 2023-10-09T06:02:09.479532+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.125:0/83317155' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:09 smithi125 bash[33919]: audit 2023-10-09T06:02:09.561546+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:09 smithi155 bash[27817]: cluster 2023-10-09T06:02:08.224764+0000 mgr.y (mgr.14430) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:09 smithi155 bash[27817]: audit 2023-10-09T06:02:09.426922+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:09 smithi155 bash[27817]: audit 2023-10-09T06:02:09.479532+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.125:0/83317155' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:09 smithi155 bash[27817]: audit 2023-10-09T06:02:09.561546+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:02:10.276 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":18,"active_gid":14430,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":26344715},{"type":"v1","addr":"172.21.15.125:6801","nonce":26344715}]},"active_addr":"172.21.15.125:6801/26344715","active_change":"2023-10-09T05:59:24.160779+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":4016621680}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":457536542}]}]} 2023-10-09T06:02:10.280 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T06:02:10.280 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:02:10.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:02:11.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:11 smithi155 bash[27817]: cluster 2023-10-09T06:02:10.225475+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:11.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:11 smithi125 bash[27513]: cluster 2023-10-09T06:02:10.225475+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:11.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:11 smithi125 bash[33919]: cluster 2023-10-09T06:02:10.225475+0000 mgr.y (mgr.14430) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:13.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:13 smithi125 bash[27513]: cluster 2023-10-09T06:02:12.226660+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:13 smithi125 bash[33919]: cluster 2023-10-09T06:02:12.226660+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:13 smithi155 bash[27817]: cluster 2023-10-09T06:02:12.226660+0000 mgr.y (mgr.14430) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:15.055 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:15 smithi125 bash[27513]: cluster 2023-10-09T06:02:14.227469+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:15 smithi125 bash[33919]: cluster 2023-10-09T06:02:14.227469+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:15 smithi155 bash[27817]: cluster 2023-10-09T06:02:14.227469+0000 mgr.y (mgr.14430) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:17.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:17 smithi125 bash[27513]: cluster 2023-10-09T06:02:16.228560+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:17 smithi125 bash[33919]: cluster 2023-10-09T06:02:16.228560+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:17 smithi155 bash[27817]: cluster 2023-10-09T06:02:16.228560+0000 mgr.y (mgr.14430) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T06:02:18.168 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:02:18.168 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":60,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:01:52.661804+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:02:18.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:18 smithi125 bash[27513]: audit 2023-10-09T06:02:18.165003+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.125:0/2738240448' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:18.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:18 smithi125 bash[33919]: audit 2023-10-09T06:02:18.165003+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.125:0/2738240448' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:18.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:18] "GET /metrics HTTP/1.1" 200 191497 "" "Prometheus/2.33.4" 2023-10-09T06:02:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:18 smithi155 bash[27817]: audit 2023-10-09T06:02:18.165003+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.125:0/2738240448' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:18.963 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:02:18.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:02:19.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:19 smithi125 bash[27513]: cluster 2023-10-09T06:02:18.228954+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:19 smithi125 bash[33919]: cluster 2023-10-09T06:02:18.228954+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:19 smithi155 bash[27817]: cluster 2023-10-09T06:02:18.228954+0000 mgr.y (mgr.14430) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:21.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:21 smithi125 bash[27513]: cluster 2023-10-09T06:02:20.229529+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:21 smithi125 bash[33919]: cluster 2023-10-09T06:02:20.229529+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:21 smithi155 bash[27817]: cluster 2023-10-09T06:02:20.229529+0000 mgr.y (mgr.14430) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:21.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:23.740 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:23 smithi155 bash[27817]: cluster 2023-10-09T06:02:22.230614+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:24.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:23 smithi125 bash[27513]: cluster 2023-10-09T06:02:22.230614+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:24.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:23 smithi125 bash[33919]: cluster 2023-10-09T06:02:22.230614+0000 mgr.y (mgr.14430) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.211927+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.212228+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.212427+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.212771+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.213820+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[27513]: audit 2023-10-09T06:02:24.214327+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.211927+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.212228+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.212427+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.212771+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.213820+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:24.920 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:24 smithi125 bash[33919]: audit 2023-10-09T06:02:24.214327+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:25.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.211927+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:25.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.212228+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:25.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.212427+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:25.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.212771+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-10-09T06:02:25.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.213820+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:02:25.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:24 smithi155 bash[27817]: audit 2023-10-09T06:02:24.214327+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T06:02:25.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[27513]: cluster 2023-10-09T06:02:24.231110+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[27513]: audit 2023-10-09T06:02:24.648319+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[27513]: audit 2023-10-09T06:02:24.648385+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[27513]: audit 2023-10-09T06:02:24.648429+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[27513]: cluster 2023-10-09T06:02:24.648469+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[33919]: cluster 2023-10-09T06:02:24.231110+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[33919]: audit 2023-10-09T06:02:24.648319+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:02:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[33919]: audit 2023-10-09T06:02:24.648385+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:02:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[33919]: audit 2023-10-09T06:02:24.648429+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T06:02:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:25 smithi125 bash[33919]: cluster 2023-10-09T06:02:24.648469+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T06:02:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:25 smithi155 bash[27817]: cluster 2023-10-09T06:02:24.231110+0000 mgr.y (mgr.14430) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:25 smithi155 bash[27817]: audit 2023-10-09T06:02:24.648319+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-10-09T06:02:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:25 smithi155 bash[27817]: audit 2023-10-09T06:02:24.648385+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:02:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:25 smithi155 bash[27817]: audit 2023-10-09T06:02:24.648429+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T06:02:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:25 smithi155 bash[27817]: cluster 2023-10-09T06:02:24.648469+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-09T06:02:26.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:26 smithi125 bash[33919]: cluster 2023-10-09T06:02:25.653733+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T06:02:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:26 smithi125 bash[33919]: cluster 2023-10-09T06:02:26.231652+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:26 smithi125 bash[27513]: cluster 2023-10-09T06:02:25.653733+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T06:02:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:26 smithi125 bash[27513]: cluster 2023-10-09T06:02:26.231652+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:26.992 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:02:26.993 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":62,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:02:25.642898+0000","last_up_change":"2023-10-09T05:58:50.307206+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6803","nonce":3657277200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6805","nonce":3657277200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6809","nonce":3657277200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":3657277200},{"type":"v1","addr":"172.21.15.125:6807","nonce":3657277200}]},"public_addr":"172.21.15.125:6803/3657277200","cluster_addr":"172.21.15.125:6805/3657277200","heartbeat_back_addr":"172.21.15.125:6809/3657277200","heartbeat_front_addr":"172.21.15.125:6807/3657277200","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6811","nonce":753846570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6813","nonce":753846570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6817","nonce":753846570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":753846570},{"type":"v1","addr":"172.21.15.125:6815","nonce":753846570}]},"public_addr":"172.21.15.125:6811/753846570","cluster_addr":"172.21.15.125:6813/753846570","heartbeat_back_addr":"172.21.15.125:6817/753846570","heartbeat_front_addr":"172.21.15.125:6815/753846570","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6819","nonce":3371306647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6821","nonce":3371306647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6825","nonce":3371306647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":3371306647},{"type":"v1","addr":"172.21.15.125:6823","nonce":3371306647}]},"public_addr":"172.21.15.125:6819/3371306647","cluster_addr":"172.21.15.125:6821/3371306647","heartbeat_back_addr":"172.21.15.125:6825/3371306647","heartbeat_front_addr":"172.21.15.125:6823/3371306647","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6827","nonce":909255933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6829","nonce":909255933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6833","nonce":909255933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":909255933},{"type":"v1","addr":"172.21.15.125:6831","nonce":909255933}]},"public_addr":"172.21.15.125:6827/909255933","cluster_addr":"172.21.15.125:6829/909255933","heartbeat_back_addr":"172.21.15.125:6833/909255933","heartbeat_front_addr":"172.21.15.125:6831/909255933","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:02:27.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:26 smithi155 bash[27817]: cluster 2023-10-09T06:02:25.653733+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-09T06:02:27.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:26 smithi155 bash[27817]: cluster 2023-10-09T06:02:26.231652+0000 mgr.y (mgr.14430) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:27.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:02:27.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:02:27.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:02:27.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:02:27.855 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:02:27.855 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:02:27.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:02:27.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:02:28.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:27 smithi125 bash[27513]: audit 2023-10-09T06:02:26.988353+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.125:0/2287579781' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:28.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:27 smithi125 bash[33919]: audit 2023-10-09T06:02:26.988353+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.125:0/2287579781' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:28.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:27 smithi155 bash[27817]: audit 2023-10-09T06:02:26.988353+0000 mon.c (mon.2) 89 : audit [DBG] from='client.? 172.21.15.125:0/2287579781' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:02:29.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:28] "GET /metrics HTTP/1.1" 200 191497 "" "Prometheus/2.33.4" 2023-10-09T06:02:29.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:28 smithi125 bash[33919]: cluster 2023-10-09T06:02:28.232282+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:29.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:28 smithi125 bash[27513]: cluster 2023-10-09T06:02:28.232282+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:29.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:28 smithi155 bash[27817]: cluster 2023-10-09T06:02:28.232282+0000 mgr.y (mgr.14430) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:31.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:31 smithi125 bash[27513]: cluster 2023-10-09T06:02:30.233177+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:31 smithi125 bash[33919]: cluster 2023-10-09T06:02:30.233177+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:31 smithi155 bash[27817]: cluster 2023-10-09T06:02:30.233177+0000 mgr.y (mgr.14430) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:32.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:31 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:32.742 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.743 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.743 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.743 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.744 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.744 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.744 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:32.751 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:33 smithi155 bash[27817]: cluster 2023-10-09T06:02:32.233834+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:33.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:33 smithi125 bash[33919]: cluster 2023-10-09T06:02:32.233834+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:33 smithi125 bash[27513]: cluster 2023-10-09T06:02:32.233834+0000 mgr.y (mgr.14430) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:35.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:35 smithi125 bash[27513]: cluster 2023-10-09T06:02:34.234524+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:35.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:35 smithi125 bash[33919]: cluster 2023-10-09T06:02:34.234524+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:35 smithi155 bash[27817]: cluster 2023-10-09T06:02:34.234524+0000 mgr.y (mgr.14430) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:37.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:37 smithi125 bash[33919]: cluster 2023-10-09T06:02:36.235611+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:37.622 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:37 smithi125 bash[27513]: cluster 2023-10-09T06:02:36.235611+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:37 smithi155 bash[27817]: cluster 2023-10-09T06:02:36.235611+0000 mgr.y (mgr.14430) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:38.967 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:38 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:38] "GET /metrics HTTP/1.1" 200 191497 "" "Prometheus/2.33.4" 2023-10-09T06:02:39.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:39 smithi125 bash[27513]: cluster 2023-10-09T06:02:38.236287+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:39.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:39 smithi125 bash[33919]: cluster 2023-10-09T06:02:38.236287+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:39 smithi155 bash[27817]: cluster 2023-10-09T06:02:38.236287+0000 mgr.y (mgr.14430) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:41.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:41 smithi125 bash[27513]: cluster 2023-10-09T06:02:40.236990+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:41.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:41 smithi125 bash[33919]: cluster 2023-10-09T06:02:40.236990+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:41.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:41 smithi155 bash[27817]: cluster 2023-10-09T06:02:40.236990+0000 mgr.y (mgr.14430) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:42.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:41 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:43.711 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:43 smithi125 bash[27513]: cluster 2023-10-09T06:02:42.237554+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:43.711 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:43 smithi125 bash[33919]: cluster 2023-10-09T06:02:42.237554+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:43 smithi155 bash[27817]: cluster 2023-10-09T06:02:42.237554+0000 mgr.y (mgr.14430) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:45.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:45 smithi125 bash[27513]: cluster 2023-10-09T06:02:44.238342+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:45.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:45 smithi125 bash[33919]: cluster 2023-10-09T06:02:44.238342+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:45 smithi155 bash[27817]: cluster 2023-10-09T06:02:44.238342+0000 mgr.y (mgr.14430) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:46.111 INFO:teuthology.orchestra.run.smithi125.stdout:128849018955 2023-10-09T06:02:46.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:02:46.750 INFO:teuthology.orchestra.run.smithi125.stdout:107374182483 2023-10-09T06:02:46.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:02:47.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:47 smithi125 bash[27513]: cluster 2023-10-09T06:02:46.239174+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:47.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:47 smithi125 bash[33919]: cluster 2023-10-09T06:02:46.239174+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:47 smithi155 bash[27817]: cluster 2023-10-09T06:02:46.239174+0000 mgr.y (mgr.14430) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:48.474 INFO:teuthology.orchestra.run.smithi125.stdout:180388626489 2023-10-09T06:02:48.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:02:48.819 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:48] "GET /metrics HTTP/1.1" 200 191530 "" "Prometheus/2.33.4" 2023-10-09T06:02:48.985 INFO:teuthology.orchestra.run.smithi125.stdout:55834574950 2023-10-09T06:02:48.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:02:49.496 INFO:teuthology.orchestra.run.smithi125.stdout:206158430258 2023-10-09T06:02:49.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:02:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:49 smithi125 bash[27513]: cluster 2023-10-09T06:02:48.239655+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:49.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:49 smithi125 bash[33919]: cluster 2023-10-09T06:02:48.239655+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:49 smithi155 bash[27817]: cluster 2023-10-09T06:02:48.239655+0000 mgr.y (mgr.14430) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:51.091 INFO:teuthology.orchestra.run.smithi125.stdout:34359738479 2023-10-09T06:02:51.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:02:51.107 INFO:teuthology.orchestra.run.smithi125.stdout:77309411422 2023-10-09T06:02:51.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:02:51.125 INFO:teuthology.orchestra.run.smithi125.stdout:154618822723 2023-10-09T06:02:51.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:02:51.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:51 smithi155 bash[27817]: cluster 2023-10-09T06:02:50.240173+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:51.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:51 smithi125 bash[27513]: cluster 2023-10-09T06:02:50.240173+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:51.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:51 smithi125 bash[33919]: cluster 2023-10-09T06:02:50.240173+0000 mgr.y (mgr.14430) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:02:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:02:53.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:53 smithi125 bash[27513]: cluster 2023-10-09T06:02:52.240753+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:53.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:53 smithi125 bash[33919]: cluster 2023-10-09T06:02:52.240753+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:53 smithi155 bash[27817]: cluster 2023-10-09T06:02:52.240753+0000 mgr.y (mgr.14430) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:53.903 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:53.904 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:53.904 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:53.904 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:53.905 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:54.911 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:55.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:55 smithi125 bash[27513]: cluster 2023-10-09T06:02:54.241395+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:55 smithi125 bash[33919]: cluster 2023-10-09T06:02:54.241395+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:55 smithi155 bash[27817]: cluster 2023-10-09T06:02:54.241395+0000 mgr.y (mgr.14430) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:55.927 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:55.928 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:02:57.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:57 smithi125 bash[27513]: cluster 2023-10-09T06:02:56.242561+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:57.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:57 smithi125 bash[33919]: cluster 2023-10-09T06:02:56.242561+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:57 smithi155 bash[27817]: cluster 2023-10-09T06:02:56.242561+0000 mgr.y (mgr.14430) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:58.804 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:02:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:02:58] "GET /metrics HTTP/1.1" 200 191523 "" "Prometheus/2.33.4" 2023-10-09T06:02:59.576 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:02:59 smithi125 bash[27513]: cluster 2023-10-09T06:02:58.243106+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:02:59 smithi155 bash[27817]: cluster 2023-10-09T06:02:58.243106+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:02:59.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:02:59 smithi125 bash[33919]: cluster 2023-10-09T06:02:58.243106+0000 mgr.y (mgr.14430) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:01 smithi125 bash[27513]: cluster 2023-10-09T06:03:00.243936+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:01 smithi155 bash[27817]: cluster 2023-10-09T06:03:00.243936+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:01.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:01 smithi125 bash[33919]: cluster 2023-10-09T06:03:00.243936+0000 mgr.y (mgr.14430) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:01 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:03 smithi125 bash[27513]: cluster 2023-10-09T06:03:02.244553+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:03 smithi125 bash[33919]: cluster 2023-10-09T06:03:02.244553+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:03 smithi155 bash[27817]: cluster 2023-10-09T06:03:02.244553+0000 mgr.y (mgr.14430) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:04 smithi155 bash[27817]: audit 2023-10-09T06:03:04.167786+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:04 smithi155 bash[27817]: audit 2023-10-09T06:03:04.428943+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:04.972 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:04 smithi125 bash[27513]: audit 2023-10-09T06:03:04.167786+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:04.973 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:04 smithi125 bash[27513]: audit 2023-10-09T06:03:04.428943+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:04.973 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:04 smithi125 bash[33919]: audit 2023-10-09T06:03:04.167786+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:04.973 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:04 smithi125 bash[33919]: audit 2023-10-09T06:03:04.428943+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:05.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[33919]: cluster 2023-10-09T06:03:04.245159+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[33919]: audit 2023-10-09T06:03:04.709640+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[33919]: audit 2023-10-09T06:03:04.711309+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[33919]: audit 2023-10-09T06:03:04.718074+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[27513]: cluster 2023-10-09T06:03:04.245159+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[27513]: audit 2023-10-09T06:03:04.709640+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[27513]: audit 2023-10-09T06:03:04.711309+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:05.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:05 smithi125 bash[27513]: audit 2023-10-09T06:03:04.718074+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:05 smithi155 bash[27817]: cluster 2023-10-09T06:03:04.245159+0000 mgr.y (mgr.14430) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:05 smithi155 bash[27817]: audit 2023-10-09T06:03:04.709640+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:05 smithi155 bash[27817]: audit 2023-10-09T06:03:04.711309+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:05 smithi155 bash[27817]: audit 2023-10-09T06:03:04.718074+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:07.869 INFO:teuthology.orchestra.run.smithi125.stdout:206158430262 2023-10-09T06:03:07.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:07 smithi125 bash[27513]: cluster 2023-10-09T06:03:06.246232+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:07.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:07 smithi125 bash[33919]: cluster 2023-10-09T06:03:06.246232+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:07 smithi155 bash[27817]: cluster 2023-10-09T06:03:06.246232+0000 mgr.y (mgr.14430) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:07.939 INFO:teuthology.orchestra.run.smithi125.stdout:107374182488 2023-10-09T06:03:08.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:08 smithi125 bash[27513]: audit 2023-10-09T06:03:07.868171+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.125:0/1981647664' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:03:08.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:08 smithi125 bash[27513]: audit 2023-10-09T06:03:07.938456+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.125:0/594926653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:03:08.748 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:08 smithi125 bash[33919]: audit 2023-10-09T06:03:07.868171+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.125:0/1981647664' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:03:08.748 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:08 smithi125 bash[33919]: audit 2023-10-09T06:03:07.938456+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.125:0/594926653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:03:08.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:08 smithi155 bash[27817]: audit 2023-10-09T06:03:07.868171+0000 mon.c (mon.2) 93 : audit [DBG] from='client.? 172.21.15.125:0/1981647664' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:03:08.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:08 smithi155 bash[27817]: audit 2023-10-09T06:03:07.938456+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.125:0/594926653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:03:09.099 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:08] "GET /metrics HTTP/1.1" 200 191523 "" "Prometheus/2.33.4" 2023-10-09T06:03:09.409 INFO:teuthology.orchestra.run.smithi125.stdout:55834574955 2023-10-09T06:03:09.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:09 smithi125 bash[27513]: cluster 2023-10-09T06:03:08.246872+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:09.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:09 smithi125 bash[27513]: audit 2023-10-09T06:03:09.407687+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.125:0/3078283786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:03:09.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:09 smithi125 bash[33919]: cluster 2023-10-09T06:03:08.246872+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:09.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:09 smithi125 bash[33919]: audit 2023-10-09T06:03:09.407687+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.125:0/3078283786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:03:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:09 smithi155 bash[27817]: cluster 2023-10-09T06:03:08.246872+0000 mgr.y (mgr.14430) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:09 smithi155 bash[27817]: audit 2023-10-09T06:03:09.407687+0000 mon.c (mon.2) 94 : audit [DBG] from='client.? 172.21.15.125:0/3078283786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:03:10.102 INFO:teuthology.orchestra.run.smithi125.stdout:180388626494 2023-10-09T06:03:10.331 INFO:teuthology.orchestra.run.smithi125.stdout:34359738483 2023-10-09T06:03:10.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:10 smithi125 bash[27513]: audit 2023-10-09T06:03:10.100656+0000 mon.a (mon.0) 683 : audit [DBG] from='client.? 172.21.15.125:0/3574126325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:03:10.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:10 smithi125 bash[27513]: audit 2023-10-09T06:03:10.329656+0000 mon.a (mon.0) 684 : audit [DBG] from='client.? 172.21.15.125:0/2268200339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:03:10.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:10 smithi125 bash[33919]: audit 2023-10-09T06:03:10.100656+0000 mon.a (mon.0) 683 : audit [DBG] from='client.? 172.21.15.125:0/3574126325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:03:10.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:10 smithi125 bash[33919]: audit 2023-10-09T06:03:10.329656+0000 mon.a (mon.0) 684 : audit [DBG] from='client.? 172.21.15.125:0/2268200339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:03:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:10 smithi155 bash[27817]: audit 2023-10-09T06:03:10.100656+0000 mon.a (mon.0) 683 : audit [DBG] from='client.? 172.21.15.125:0/3574126325' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:03:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:10 smithi155 bash[27817]: audit 2023-10-09T06:03:10.329656+0000 mon.a (mon.0) 684 : audit [DBG] from='client.? 172.21.15.125:0/2268200339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:11 smithi125 bash[27513]: cluster 2023-10-09T06:03:10.247392+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:11.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:11 smithi125 bash[33919]: cluster 2023-10-09T06:03:10.247392+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:11.831 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574950 got 55834574955 for osd.1 2023-10-09T06:03:11.831 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:11.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:11 smithi155 bash[27817]: cluster 2023-10-09T06:03:10.247392+0000 mgr.y (mgr.14430) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:12.352 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430258 got 206158430262 for osd.7 2023-10-09T06:03:12.352 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:13.122 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182483 got 107374182488 for osd.3 2023-10-09T06:03:13.122 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:13.660 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626489 got 180388626494 for osd.6 2023-10-09T06:03:13.660 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:13 smithi155 bash[27817]: cluster 2023-10-09T06:03:12.247920+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:13.895 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:13 smithi125 bash[27513]: cluster 2023-10-09T06:03:12.247920+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:13.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:13 smithi125 bash[33919]: cluster 2023-10-09T06:03:12.247920+0000 mgr.y (mgr.14430) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:14.008 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738479 got 34359738483 for osd.0 2023-10-09T06:03:14.008 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:14.310 INFO:teuthology.orchestra.run.smithi125.stdout:77309411427 2023-10-09T06:03:14.327 INFO:teuthology.orchestra.run.smithi125.stdout:128849018961 2023-10-09T06:03:14.476 INFO:teuthology.orchestra.run.smithi125.stdout:154618822728 2023-10-09T06:03:14.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[27513]: audit 2023-10-09T06:03:14.308410+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.125:0/1768518510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:03:14.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[27513]: audit 2023-10-09T06:03:14.325988+0000 mon.a (mon.0) 685 : audit [DBG] from='client.? 172.21.15.125:0/1981472262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:03:14.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[27513]: audit 2023-10-09T06:03:14.474927+0000 mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.125:0/3249962431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:03:14.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[33919]: audit 2023-10-09T06:03:14.308410+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.125:0/1768518510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:03:14.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[33919]: audit 2023-10-09T06:03:14.325988+0000 mon.a (mon.0) 685 : audit [DBG] from='client.? 172.21.15.125:0/1981472262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:03:14.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:14 smithi125 bash[33919]: audit 2023-10-09T06:03:14.474927+0000 mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.125:0/3249962431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:03:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:14 smithi155 bash[27817]: audit 2023-10-09T06:03:14.308410+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.125:0/1768518510' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:03:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:14 smithi155 bash[27817]: audit 2023-10-09T06:03:14.325988+0000 mon.a (mon.0) 685 : audit [DBG] from='client.? 172.21.15.125:0/1981472262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:03:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:14 smithi155 bash[27817]: audit 2023-10-09T06:03:14.474927+0000 mon.a (mon.0) 686 : audit [DBG] from='client.? 172.21.15.125:0/3249962431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:03:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:15 smithi125 bash[27513]: cluster 2023-10-09T06:03:14.248458+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:15 smithi125 bash[33919]: cluster 2023-10-09T06:03:14.248458+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:15 smithi155 bash[27817]: cluster 2023-10-09T06:03:14.248458+0000 mgr.y (mgr.14430) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:16.135 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411422 got 77309411427 for osd.2 2023-10-09T06:03:16.136 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:16.317 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018955 got 128849018961 for osd.4 2023-10-09T06:03:16.317 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:16.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822723 got 154618822728 for osd.5 2023-10-09T06:03:16.460 DEBUG:teuthology.parallel:result is None 2023-10-09T06:03:16.460 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:03:16.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:03:17.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:17 smithi125 bash[27513]: cluster 2023-10-09T06:03:16.249538+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:17 smithi125 bash[33919]: cluster 2023-10-09T06:03:16.249538+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:17 smithi155 bash[27817]: cluster 2023-10-09T06:03:16.249538+0000 mgr.y (mgr.14430) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:18] "GET /metrics HTTP/1.1" 200 191534 "" "Prometheus/2.33.4" 2023-10-09T06:03:19.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:19 smithi125 bash[27513]: cluster 2023-10-09T06:03:18.250157+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:19 smithi125 bash[33919]: cluster 2023-10-09T06:03:18.250157+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:19 smithi155 bash[27817]: cluster 2023-10-09T06:03:18.250157+0000 mgr.y (mgr.14430) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:21.237 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:21.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:21 smithi155 bash[27817]: cluster 2023-10-09T06:03:20.251105+0000 mgr.y (mgr.14430) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:21.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:21 smithi125 bash[27513]: cluster 2023-10-09T06:03:20.251105+0000 mgr.y (mgr.14430) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:21.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:21 smithi125 bash[33919]: cluster 2023-10-09T06:03:20.251105+0000 mgr.y (mgr.14430) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:23.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:23 smithi125 bash[27513]: cluster 2023-10-09T06:03:22.251486+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:23 smithi125 bash[33919]: cluster 2023-10-09T06:03:22.251486+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:23 smithi155 bash[27817]: cluster 2023-10-09T06:03:22.251486+0000 mgr.y (mgr.14430) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:24.422 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:03:24.423 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:03:25.205 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":134,"stamp":"2023-10-09T06:03:24.251747+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381740,"kb_used_data":4972,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579476,"statfs":{"total":767960285184,"available":765521383424,"internally_reserved":0,"allocated":5091328,"data_stored":2164863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004167"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647513+0000","last_change":"2023-10-09T06:01:51.474967+0000","last_active":"2023-10-09T06:02:24.647513+0000","last_peered":"2023-10-09T06:02:24.647513+0000","last_clean":"2023-10-09T06:02:24.647513+0000","last_became_active":"2023-10-09T06:01:51.473515+0000","last_became_peered":"2023-10-09T06:01:51.473515+0000","last_unstale":"2023-10-09T06:02:24.647513+0000","last_undegraded":"2023-10-09T06:02:24.647513+0000","last_fullsized":"2023-10-09T06:02:24.647513+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.646361+0000","last_change":"2023-10-09T06:01:50.649268+0000","last_active":"2023-10-09T06:02:24.646361+0000","last_peered":"2023-10-09T06:02:24.646361+0000","last_clean":"2023-10-09T06:02:24.646361+0000","last_became_active":"2023-10-09T06:01:50.648924+0000","last_became_peered":"2023-10-09T06:01:50.648924+0000","last_unstale":"2023-10-09T06:02:24.646361+0000","last_undegraded":"2023-10-09T06:02:24.646361+0000","last_fullsized":"2023-10-09T06:02:24.646361+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651639+0000","last_change":"2023-10-09T06:01:50.672522+0000","last_active":"2023-10-09T06:02:24.651639+0000","last_peered":"2023-10-09T06:02:24.651639+0000","last_clean":"2023-10-09T06:02:24.651639+0000","last_became_active":"2023-10-09T06:01:50.672103+0000","last_became_peered":"2023-10-09T06:01:50.672103+0000","last_unstale":"2023-10-09T06:02:24.651639+0000","last_undegraded":"2023-10-09T06:02:24.651639+0000","last_fullsized":"2023-10-09T06:02:24.651639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652145+0000","last_change":"2023-10-09T06:01:50.649515+0000","last_active":"2023-10-09T06:02:24.652145+0000","last_peered":"2023-10-09T06:02:24.652145+0000","last_clean":"2023-10-09T06:02:24.652145+0000","last_became_active":"2023-10-09T06:01:50.649150+0000","last_became_peered":"2023-10-09T06:01:50.649150+0000","last_unstale":"2023-10-09T06:02:24.652145+0000","last_undegraded":"2023-10-09T06:02:24.652145+0000","last_fullsized":"2023-10-09T06:02:24.652145+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647973+0000","last_change":"2023-10-09T06:01:50.673776+0000","last_active":"2023-10-09T06:02:24.647973+0000","last_peered":"2023-10-09T06:02:24.647973+0000","last_clean":"2023-10-09T06:02:24.647973+0000","last_became_active":"2023-10-09T06:01:50.673445+0000","last_became_peered":"2023-10-09T06:01:50.673445+0000","last_unstale":"2023-10-09T06:02:24.647973+0000","last_undegraded":"2023-10-09T06:02:24.647973+0000","last_fullsized":"2023-10-09T06:02:24.647973+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657125+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:02:24.657125+0000","last_peered":"2023-10-09T06:02:24.657125+0000","last_clean":"2023-10-09T06:02:24.657125+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:02:24.657125+0000","last_undegraded":"2023-10-09T06:02:24.657125+0000","last_fullsized":"2023-10-09T06:02:24.657125+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648152+0000","last_change":"2023-10-09T06:01:51.478485+0000","last_active":"2023-10-09T06:02:24.648152+0000","last_peered":"2023-10-09T06:02:24.648152+0000","last_clean":"2023-10-09T06:02:24.648152+0000","last_became_active":"2023-10-09T06:01:51.477533+0000","last_became_peered":"2023-10-09T06:01:51.477533+0000","last_unstale":"2023-10-09T06:02:24.648152+0000","last_undegraded":"2023-10-09T06:02:24.648152+0000","last_fullsized":"2023-10-09T06:02:24.648152+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657000+0000","last_change":"2023-10-09T06:01:50.652132+0000","last_active":"2023-10-09T06:02:24.657000+0000","last_peered":"2023-10-09T06:02:24.657000+0000","last_clean":"2023-10-09T06:02:24.657000+0000","last_became_active":"2023-10-09T06:01:50.651802+0000","last_became_peered":"2023-10-09T06:01:50.651802+0000","last_unstale":"2023-10-09T06:02:24.657000+0000","last_undegraded":"2023-10-09T06:02:24.657000+0000","last_fullsized":"2023-10-09T06:02:24.657000+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.536637+0000","last_change":"2023-10-09T06:02:26.536637+0000","last_active":"2023-10-09T06:02:26.536637+0000","last_peered":"2023-10-09T06:02:26.536637+0000","last_clean":"2023-10-09T06:02:26.536637+0000","last_became_active":"2023-10-09T06:02:26.535915+0000","last_became_peered":"2023-10-09T06:02:26.535915+0000","last_unstale":"2023-10-09T06:02:26.536637+0000","last_undegraded":"2023-10-09T06:02:26.536637+0000","last_fullsized":"2023-10-09T06:02:26.536637+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645719+0000","last_change":"2023-10-09T06:01:50.650579+0000","last_active":"2023-10-09T06:02:24.645719+0000","last_peered":"2023-10-09T06:02:24.645719+0000","last_clean":"2023-10-09T06:02:24.645719+0000","last_became_active":"2023-10-09T06:01:50.650246+0000","last_became_peered":"2023-10-09T06:01:50.650246+0000","last_unstale":"2023-10-09T06:02:24.645719+0000","last_undegraded":"2023-10-09T06:02:24.645719+0000","last_fullsized":"2023-10-09T06:02:24.645719+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647940+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:02:24.647940+0000","last_peered":"2023-10-09T06:02:24.647940+0000","last_clean":"2023-10-09T06:02:24.647940+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:02:24.647940+0000","last_undegraded":"2023-10-09T06:02:24.647940+0000","last_fullsized":"2023-10-09T06:02:24.647940+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645917+0000","last_change":"2023-10-09T06:01:50.648001+0000","last_active":"2023-10-09T06:02:24.645917+0000","last_peered":"2023-10-09T06:02:24.645917+0000","last_clean":"2023-10-09T06:02:24.645917+0000","last_became_active":"2023-10-09T06:01:50.647686+0000","last_became_peered":"2023-10-09T06:01:50.647686+0000","last_unstale":"2023-10-09T06:02:24.645917+0000","last_undegraded":"2023-10-09T06:02:24.645917+0000","last_fullsized":"2023-10-09T06:02:24.645917+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655290+0000","last_change":"2023-10-09T05:59:52.469538+0000","last_active":"2023-10-09T06:02:24.655290+0000","last_peered":"2023-10-09T06:02:24.655290+0000","last_clean":"2023-10-09T06:02:24.655290+0000","last_became_active":"2023-10-09T05:59:52.469011+0000","last_became_peered":"2023-10-09T05:59:52.469011+0000","last_unstale":"2023-10-09T06:02:24.655290+0000","last_undegraded":"2023-10-09T06:02:24.655290+0000","last_fullsized":"2023-10-09T06:02:24.655290+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651405+0000","last_change":"2023-10-09T06:01:50.678265+0000","last_active":"2023-10-09T06:02:24.651405+0000","last_peered":"2023-10-09T06:02:24.651405+0000","last_clean":"2023-10-09T06:02:24.651405+0000","last_became_active":"2023-10-09T06:01:50.677885+0000","last_became_peered":"2023-10-09T06:01:50.677885+0000","last_unstale":"2023-10-09T06:02:24.651405+0000","last_undegraded":"2023-10-09T06:02:24.651405+0000","last_fullsized":"2023-10-09T06:02:24.651405+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655952+0000","last_change":"2023-10-09T05:59:52.779228+0000","last_active":"2023-10-09T06:02:24.655952+0000","last_peered":"2023-10-09T06:02:24.655952+0000","last_clean":"2023-10-09T06:02:24.655952+0000","last_became_active":"2023-10-09T05:59:52.777994+0000","last_became_peered":"2023-10-09T05:59:52.777994+0000","last_unstale":"2023-10-09T06:02:24.655952+0000","last_undegraded":"2023-10-09T06:02:24.655952+0000","last_fullsized":"2023-10-09T06:02:24.655952+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652312+0000","last_change":"2023-10-09T06:01:50.656699+0000","last_active":"2023-10-09T06:02:24.652312+0000","last_peered":"2023-10-09T06:02:24.652312+0000","last_clean":"2023-10-09T06:02:24.652312+0000","last_became_active":"2023-10-09T06:01:50.656372+0000","last_became_peered":"2023-10-09T06:01:50.656372+0000","last_unstale":"2023-10-09T06:02:24.652312+0000","last_undegraded":"2023-10-09T06:02:24.652312+0000","last_fullsized":"2023-10-09T06:02:24.652312+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645134+0000","last_change":"2023-10-09T06:01:50.678079+0000","last_active":"2023-10-09T06:02:24.645134+0000","last_peered":"2023-10-09T06:02:24.645134+0000","last_clean":"2023-10-09T06:02:24.645134+0000","last_became_active":"2023-10-09T06:01:50.677755+0000","last_became_peered":"2023-10-09T06:01:50.677755+0000","last_unstale":"2023-10-09T06:02:24.645134+0000","last_undegraded":"2023-10-09T06:02:24.645134+0000","last_fullsized":"2023-10-09T06:02:24.645134+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656309+0000","last_change":"2023-10-09T06:01:50.655267+0000","last_active":"2023-10-09T06:02:24.656309+0000","last_peered":"2023-10-09T06:02:24.656309+0000","last_clean":"2023-10-09T06:02:24.656309+0000","last_became_active":"2023-10-09T06:01:50.654934+0000","last_became_peered":"2023-10-09T06:01:50.654934+0000","last_unstale":"2023-10-09T06:02:24.656309+0000","last_undegraded":"2023-10-09T06:02:24.656309+0000","last_fullsized":"2023-10-09T06:02:24.656309+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649109+0000","last_change":"2023-10-09T05:59:52.778611+0000","last_active":"2023-10-09T06:02:24.649109+0000","last_peered":"2023-10-09T06:02:24.649109+0000","last_clean":"2023-10-09T06:02:24.649109+0000","last_became_active":"2023-10-09T05:59:52.777893+0000","last_became_peered":"2023-10-09T05:59:52.777893+0000","last_unstale":"2023-10-09T06:02:24.649109+0000","last_undegraded":"2023-10-09T06:02:24.649109+0000","last_fullsized":"2023-10-09T06:02:24.649109+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645925+0000","last_change":"2023-10-09T06:01:51.475796+0000","last_active":"2023-10-09T06:02:24.645925+0000","last_peered":"2023-10-09T06:02:24.645925+0000","last_clean":"2023-10-09T06:02:24.645925+0000","last_became_active":"2023-10-09T06:01:51.475387+0000","last_became_peered":"2023-10-09T06:01:51.475387+0000","last_unstale":"2023-10-09T06:02:24.645925+0000","last_undegraded":"2023-10-09T06:02:24.645925+0000","last_fullsized":"2023-10-09T06:02:24.645925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645258+0000","last_change":"2023-10-09T06:01:50.658288+0000","last_active":"2023-10-09T06:02:24.645258+0000","last_peered":"2023-10-09T06:02:24.645258+0000","last_clean":"2023-10-09T06:02:24.645258+0000","last_became_active":"2023-10-09T06:01:50.657871+0000","last_became_peered":"2023-10-09T06:01:50.657871+0000","last_unstale":"2023-10-09T06:02:24.645258+0000","last_undegraded":"2023-10-09T06:02:24.645258+0000","last_fullsized":"2023-10-09T06:02:24.645258+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648108+0000","last_change":"2023-10-09T05:59:52.583913+0000","last_active":"2023-10-09T06:02:24.648108+0000","last_peered":"2023-10-09T06:02:24.648108+0000","last_clean":"2023-10-09T06:02:24.648108+0000","last_became_active":"2023-10-09T05:59:52.583298+0000","last_became_peered":"2023-10-09T05:59:52.583298+0000","last_unstale":"2023-10-09T06:02:24.648108+0000","last_undegraded":"2023-10-09T06:02:24.648108+0000","last_fullsized":"2023-10-09T06:02:24.648108+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654996+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:02:24.654996+0000","last_peered":"2023-10-09T06:02:24.654996+0000","last_clean":"2023-10-09T06:02:24.654996+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:02:24.654996+0000","last_undegraded":"2023-10-09T06:02:24.654996+0000","last_fullsized":"2023-10-09T06:02:24.654996+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651789+0000","last_change":"2023-10-09T06:01:50.650581+0000","last_active":"2023-10-09T06:02:24.651789+0000","last_peered":"2023-10-09T06:02:24.651789+0000","last_clean":"2023-10-09T06:02:24.651789+0000","last_became_active":"2023-10-09T06:01:50.650078+0000","last_became_peered":"2023-10-09T06:01:50.650078+0000","last_unstale":"2023-10-09T06:02:24.651789+0000","last_undegraded":"2023-10-09T06:02:24.651789+0000","last_fullsized":"2023-10-09T06:02:24.651789+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648015+0000","last_change":"2023-10-09T05:58:52.535269+0000","last_active":"2023-10-09T06:02:24.648015+0000","last_peered":"2023-10-09T06:02:24.648015+0000","last_clean":"2023-10-09T06:02:24.648015+0000","last_became_active":"2023-10-09T05:58:52.534584+0000","last_became_peered":"2023-10-09T05:58:52.534584+0000","last_unstale":"2023-10-09T06:02:24.648015+0000","last_undegraded":"2023-10-09T06:02:24.648015+0000","last_fullsized":"2023-10-09T06:02:24.648015+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656797+0000","last_change":"2023-10-09T05:59:52.469599+0000","last_active":"2023-10-09T06:02:24.656797+0000","last_peered":"2023-10-09T06:02:24.656797+0000","last_clean":"2023-10-09T06:02:24.656797+0000","last_became_active":"2023-10-09T05:59:52.469155+0000","last_became_peered":"2023-10-09T05:59:52.469155+0000","last_unstale":"2023-10-09T06:02:24.656797+0000","last_undegraded":"2023-10-09T06:02:24.656797+0000","last_fullsized":"2023-10-09T06:02:24.656797+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655999+0000","last_change":"2023-10-09T06:01:51.771158+0000","last_active":"2023-10-09T06:02:24.655999+0000","last_peered":"2023-10-09T06:02:24.655999+0000","last_clean":"2023-10-09T06:02:24.655999+0000","last_became_active":"2023-10-09T06:01:51.770747+0000","last_became_peered":"2023-10-09T06:01:51.770747+0000","last_unstale":"2023-10-09T06:02:24.655999+0000","last_undegraded":"2023-10-09T06:02:24.655999+0000","last_fullsized":"2023-10-09T06:02:24.655999+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656105+0000","last_change":"2023-10-09T06:01:50.653085+0000","last_active":"2023-10-09T06:02:24.656105+0000","last_peered":"2023-10-09T06:02:24.656105+0000","last_clean":"2023-10-09T06:02:24.656105+0000","last_became_active":"2023-10-09T06:01:50.652800+0000","last_became_peered":"2023-10-09T06:01:50.652800+0000","last_unstale":"2023-10-09T06:02:24.656105+0000","last_undegraded":"2023-10-09T06:02:24.656105+0000","last_fullsized":"2023-10-09T06:02:24.656105+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.653991+0000","last_change":"2023-10-09T05:59:52.778084+0000","last_active":"2023-10-09T06:02:24.653991+0000","last_peered":"2023-10-09T06:02:24.653991+0000","last_clean":"2023-10-09T06:02:24.653991+0000","last_became_active":"2023-10-09T05:59:52.777374+0000","last_became_peered":"2023-10-09T05:59:52.777374+0000","last_unstale":"2023-10-09T06:02:24.653991+0000","last_undegraded":"2023-10-09T06:02:24.653991+0000","last_fullsized":"2023-10-09T06:02:24.653991+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656223+0000","last_change":"2023-10-09T06:01:51.770156+0000","last_active":"2023-10-09T06:02:24.656223+0000","last_peered":"2023-10-09T06:02:24.656223+0000","last_clean":"2023-10-09T06:02:24.656223+0000","last_became_active":"2023-10-09T06:01:51.769850+0000","last_became_peered":"2023-10-09T06:01:51.769850+0000","last_unstale":"2023-10-09T06:02:24.656223+0000","last_undegraded":"2023-10-09T06:02:24.656223+0000","last_fullsized":"2023-10-09T06:02:24.656223+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648030+0000","last_change":"2023-10-09T06:01:50.651949+0000","last_active":"2023-10-09T06:02:24.648030+0000","last_peered":"2023-10-09T06:02:24.648030+0000","last_clean":"2023-10-09T06:02:24.648030+0000","last_became_active":"2023-10-09T06:01:50.651000+0000","last_became_peered":"2023-10-09T06:01:50.651000+0000","last_unstale":"2023-10-09T06:02:24.648030+0000","last_undegraded":"2023-10-09T06:02:24.648030+0000","last_fullsized":"2023-10-09T06:02:24.648030+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656838+0000","last_change":"2023-10-09T05:59:52.467577+0000","last_active":"2023-10-09T06:02:24.656838+0000","last_peered":"2023-10-09T06:02:24.656838+0000","last_clean":"2023-10-09T06:02:24.656838+0000","last_became_active":"2023-10-09T05:59:52.467143+0000","last_became_peered":"2023-10-09T05:59:52.467143+0000","last_unstale":"2023-10-09T06:02:24.656838+0000","last_undegraded":"2023-10-09T06:02:24.656838+0000","last_fullsized":"2023-10-09T06:02:24.656838+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655405+0000","last_change":"2023-10-09T06:01:51.770429+0000","last_active":"2023-10-09T06:02:24.655405+0000","last_peered":"2023-10-09T06:02:24.655405+0000","last_clean":"2023-10-09T06:02:24.655405+0000","last_became_active":"2023-10-09T06:01:51.769840+0000","last_became_peered":"2023-10-09T06:01:51.769840+0000","last_unstale":"2023-10-09T06:02:24.655405+0000","last_undegraded":"2023-10-09T06:02:24.655405+0000","last_fullsized":"2023-10-09T06:02:24.655405+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657062+0000","last_change":"2023-10-09T06:01:50.648057+0000","last_active":"2023-10-09T06:02:24.657062+0000","last_peered":"2023-10-09T06:02:24.657062+0000","last_clean":"2023-10-09T06:02:24.657062+0000","last_became_active":"2023-10-09T06:01:50.647717+0000","last_became_peered":"2023-10-09T06:01:50.647717+0000","last_unstale":"2023-10-09T06:02:24.657062+0000","last_undegraded":"2023-10-09T06:02:24.657062+0000","last_fullsized":"2023-10-09T06:02:24.657062+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656133+0000","last_change":"2023-10-09T05:59:52.583128+0000","last_active":"2023-10-09T06:02:24.656133+0000","last_peered":"2023-10-09T06:02:24.656133+0000","last_clean":"2023-10-09T06:02:24.656133+0000","last_became_active":"2023-10-09T05:59:52.582692+0000","last_became_peered":"2023-10-09T05:59:52.582692+0000","last_unstale":"2023-10-09T06:02:24.656133+0000","last_undegraded":"2023-10-09T06:02:24.656133+0000","last_fullsized":"2023-10-09T06:02:24.656133+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656049+0000","last_change":"2023-10-09T06:01:51.768939+0000","last_active":"2023-10-09T06:02:24.656049+0000","last_peered":"2023-10-09T06:02:24.656049+0000","last_clean":"2023-10-09T06:02:24.656049+0000","last_became_active":"2023-10-09T06:01:51.768522+0000","last_became_peered":"2023-10-09T06:01:51.768522+0000","last_unstale":"2023-10-09T06:02:24.656049+0000","last_undegraded":"2023-10-09T06:02:24.656049+0000","last_fullsized":"2023-10-09T06:02:24.656049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651456+0000","last_change":"2023-10-09T06:01:50.651743+0000","last_active":"2023-10-09T06:02:24.651456+0000","last_peered":"2023-10-09T06:02:24.651456+0000","last_clean":"2023-10-09T06:02:24.651456+0000","last_became_active":"2023-10-09T06:01:50.651429+0000","last_became_peered":"2023-10-09T06:01:50.651429+0000","last_unstale":"2023-10-09T06:02:24.651456+0000","last_undegraded":"2023-10-09T06:02:24.651456+0000","last_fullsized":"2023-10-09T06:02:24.651456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648601+0000","last_change":"2023-10-09T06:01:51.477427+0000","last_active":"2023-10-09T06:02:24.648601+0000","last_peered":"2023-10-09T06:02:24.648601+0000","last_clean":"2023-10-09T06:02:24.648601+0000","last_became_active":"2023-10-09T06:01:51.476469+0000","last_became_peered":"2023-10-09T06:01:51.476469+0000","last_unstale":"2023-10-09T06:02:24.648601+0000","last_undegraded":"2023-10-09T06:02:24.648601+0000","last_fullsized":"2023-10-09T06:02:24.648601+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648066+0000","last_change":"2023-10-09T06:01:50.653479+0000","last_active":"2023-10-09T06:02:24.648066+0000","last_peered":"2023-10-09T06:02:24.648066+0000","last_clean":"2023-10-09T06:02:24.648066+0000","last_became_active":"2023-10-09T06:01:50.653159+0000","last_became_peered":"2023-10-09T06:01:50.653159+0000","last_unstale":"2023-10-09T06:02:24.648066+0000","last_undegraded":"2023-10-09T06:02:24.648066+0000","last_fullsized":"2023-10-09T06:02:24.648066+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648793+0000","last_change":"2023-10-09T06:01:51.477429+0000","last_active":"2023-10-09T06:02:24.648793+0000","last_peered":"2023-10-09T06:02:24.648793+0000","last_clean":"2023-10-09T06:02:24.648793+0000","last_became_active":"2023-10-09T06:01:51.476565+0000","last_became_peered":"2023-10-09T06:01:51.476565+0000","last_unstale":"2023-10-09T06:02:24.648793+0000","last_undegraded":"2023-10-09T06:02:24.648793+0000","last_fullsized":"2023-10-09T06:02:24.648793+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654914+0000","last_change":"2023-10-09T06:01:50.654462+0000","last_active":"2023-10-09T06:02:24.654914+0000","last_peered":"2023-10-09T06:02:24.654914+0000","last_clean":"2023-10-09T06:02:24.654914+0000","last_became_active":"2023-10-09T06:01:50.654120+0000","last_became_peered":"2023-10-09T06:01:50.654120+0000","last_unstale":"2023-10-09T06:02:24.654914+0000","last_undegraded":"2023-10-09T06:02:24.654914+0000","last_fullsized":"2023-10-09T06:02:24.654914+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648174+0000","last_change":"2023-10-09T06:01:50.672468+0000","last_active":"2023-10-09T06:02:24.648174+0000","last_peered":"2023-10-09T06:02:24.648174+0000","last_clean":"2023-10-09T06:02:24.648174+0000","last_became_active":"2023-10-09T06:01:50.672182+0000","last_became_peered":"2023-10-09T06:01:50.672182+0000","last_unstale":"2023-10-09T06:02:24.648174+0000","last_undegraded":"2023-10-09T06:02:24.648174+0000","last_fullsized":"2023-10-09T06:02:24.648174+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651964+0000","last_change":"2023-10-09T06:01:50.655598+0000","last_active":"2023-10-09T06:02:24.651964+0000","last_peered":"2023-10-09T06:02:24.651964+0000","last_clean":"2023-10-09T06:02:24.651964+0000","last_became_active":"2023-10-09T06:01:50.655023+0000","last_became_peered":"2023-10-09T06:01:50.655023+0000","last_unstale":"2023-10-09T06:02:24.651964+0000","last_undegraded":"2023-10-09T06:02:24.651964+0000","last_fullsized":"2023-10-09T06:02:24.651964+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645525+0000","last_change":"2023-10-09T06:01:51.475825+0000","last_active":"2023-10-09T06:02:24.645525+0000","last_peered":"2023-10-09T06:02:24.645525+0000","last_clean":"2023-10-09T06:02:24.645525+0000","last_became_active":"2023-10-09T06:01:51.475392+0000","last_became_peered":"2023-10-09T06:01:51.475392+0000","last_unstale":"2023-10-09T06:02:24.645525+0000","last_undegraded":"2023-10-09T06:02:24.645525+0000","last_fullsized":"2023-10-09T06:02:24.645525+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656810+0000","last_change":"2023-10-09T06:01:50.649997+0000","last_active":"2023-10-09T06:02:24.656810+0000","last_peered":"2023-10-09T06:02:24.656810+0000","last_clean":"2023-10-09T06:02:24.656810+0000","last_became_active":"2023-10-09T06:01:50.649470+0000","last_became_peered":"2023-10-09T06:01:50.649470+0000","last_unstale":"2023-10-09T06:02:24.656810+0000","last_undegraded":"2023-10-09T06:02:24.656810+0000","last_fullsized":"2023-10-09T06:02:24.656810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654761+0000","last_change":"2023-10-09T06:01:51.475354+0000","last_active":"2023-10-09T06:02:24.654761+0000","last_peered":"2023-10-09T06:02:24.654761+0000","last_clean":"2023-10-09T06:02:24.654761+0000","last_became_active":"2023-10-09T06:01:51.474419+0000","last_became_peered":"2023-10-09T06:01:51.474419+0000","last_unstale":"2023-10-09T06:02:24.654761+0000","last_undegraded":"2023-10-09T06:02:24.654761+0000","last_fullsized":"2023-10-09T06:02:24.654761+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649163+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:02:24.649163+0000","last_peered":"2023-10-09T06:02:24.649163+0000","last_clean":"2023-10-09T06:02:24.649163+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:02:24.649163+0000","last_undegraded":"2023-10-09T06:02:24.649163+0000","last_fullsized":"2023-10-09T06:02:24.649163+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648267+0000","last_change":"2023-10-09T06:01:51.478746+0000","last_active":"2023-10-09T06:02:24.648267+0000","last_peered":"2023-10-09T06:02:24.648267+0000","last_clean":"2023-10-09T06:02:24.648267+0000","last_became_active":"2023-10-09T06:01:51.477825+0000","last_became_peered":"2023-10-09T06:01:51.477825+0000","last_unstale":"2023-10-09T06:02:24.648267+0000","last_undegraded":"2023-10-09T06:02:24.648267+0000","last_fullsized":"2023-10-09T06:02:24.648267+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649132+0000","last_change":"2023-10-09T06:01:50.649059+0000","last_active":"2023-10-09T06:02:24.649132+0000","last_peered":"2023-10-09T06:02:24.649132+0000","last_clean":"2023-10-09T06:02:24.649132+0000","last_became_active":"2023-10-09T06:01:50.648710+0000","last_became_peered":"2023-10-09T06:01:50.648710+0000","last_unstale":"2023-10-09T06:02:24.649132+0000","last_undegraded":"2023-10-09T06:02:24.649132+0000","last_fullsized":"2023-10-09T06:02:24.649132+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656688+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:02:24.656688+0000","last_peered":"2023-10-09T06:02:24.656688+0000","last_clean":"2023-10-09T06:02:24.656688+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:02:24.656688+0000","last_undegraded":"2023-10-09T06:02:24.656688+0000","last_fullsized":"2023-10-09T06:02:24.656688+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649732+0000","last_change":"2023-10-09T06:01:50.651585+0000","last_active":"2023-10-09T06:02:24.649732+0000","last_peered":"2023-10-09T06:02:24.649732+0000","last_clean":"2023-10-09T06:02:24.649732+0000","last_became_active":"2023-10-09T06:01:50.651258+0000","last_became_peered":"2023-10-09T06:01:50.651258+0000","last_unstale":"2023-10-09T06:02:24.649732+0000","last_undegraded":"2023-10-09T06:02:24.649732+0000","last_fullsized":"2023-10-09T06:02:24.649732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:25.658348+0000","last_change":"2023-10-09T06:02:25.658348+0000","last_active":"2023-10-09T06:02:25.658348+0000","last_peered":"2023-10-09T06:02:25.658348+0000","last_clean":"2023-10-09T06:02:25.658348+0000","last_became_active":"2023-10-09T06:02:25.657886+0000","last_became_peered":"2023-10-09T06:02:25.657886+0000","last_unstale":"2023-10-09T06:02:25.658348+0000","last_undegraded":"2023-10-09T06:02:25.658348+0000","last_fullsized":"2023-10-09T06:02:25.658348+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655225+0000","last_change":"2023-10-09T06:01:50.647073+0000","last_active":"2023-10-09T06:02:24.655225+0000","last_peered":"2023-10-09T06:02:24.655225+0000","last_clean":"2023-10-09T06:02:24.655225+0000","last_became_active":"2023-10-09T06:01:50.646739+0000","last_became_peered":"2023-10-09T06:01:50.646739+0000","last_unstale":"2023-10-09T06:02:24.655225+0000","last_undegraded":"2023-10-09T06:02:24.655225+0000","last_fullsized":"2023-10-09T06:02:24.655225+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645798+0000","last_change":"2023-10-09T06:01:50.672874+0000","last_active":"2023-10-09T06:02:24.645798+0000","last_peered":"2023-10-09T06:02:24.645798+0000","last_clean":"2023-10-09T06:02:24.645798+0000","last_became_active":"2023-10-09T06:01:50.672498+0000","last_became_peered":"2023-10-09T06:01:50.672498+0000","last_unstale":"2023-10-09T06:02:24.645798+0000","last_undegraded":"2023-10-09T06:02:24.645798+0000","last_fullsized":"2023-10-09T06:02:24.645798+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649238+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:02:24.649238+0000","last_peered":"2023-10-09T06:02:24.649238+0000","last_clean":"2023-10-09T06:02:24.649238+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:02:24.649238+0000","last_undegraded":"2023-10-09T06:02:24.649238+0000","last_fullsized":"2023-10-09T06:02:24.649238+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656968+0000","last_change":"2023-10-09T06:01:50.671509+0000","last_active":"2023-10-09T06:02:24.656968+0000","last_peered":"2023-10-09T06:02:24.656968+0000","last_clean":"2023-10-09T06:02:24.656968+0000","last_became_active":"2023-10-09T06:01:50.671114+0000","last_became_peered":"2023-10-09T06:01:50.671114+0000","last_unstale":"2023-10-09T06:02:24.656968+0000","last_undegraded":"2023-10-09T06:02:24.656968+0000","last_fullsized":"2023-10-09T06:02:24.656968+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645295+0000","last_change":"2023-10-09T06:01:50.658315+0000","last_active":"2023-10-09T06:02:24.645295+0000","last_peered":"2023-10-09T06:02:24.645295+0000","last_clean":"2023-10-09T06:02:24.645295+0000","last_became_active":"2023-10-09T06:01:50.657718+0000","last_became_peered":"2023-10-09T06:01:50.657718+0000","last_unstale":"2023-10-09T06:02:24.645295+0000","last_undegraded":"2023-10-09T06:02:24.645295+0000","last_fullsized":"2023-10-09T06:02:24.645295+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652349+0000","last_change":"2023-10-09T06:01:51.477867+0000","last_active":"2023-10-09T06:02:24.652349+0000","last_peered":"2023-10-09T06:02:24.652349+0000","last_clean":"2023-10-09T06:02:24.652349+0000","last_became_active":"2023-10-09T06:01:51.477333+0000","last_became_peered":"2023-10-09T06:01:51.477333+0000","last_unstale":"2023-10-09T06:02:24.652349+0000","last_undegraded":"2023-10-09T06:02:24.652349+0000","last_fullsized":"2023-10-09T06:02:24.652349+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649411+0000","last_change":"2023-10-09T06:01:50.652517+0000","last_active":"2023-10-09T06:02:24.649411+0000","last_peered":"2023-10-09T06:02:24.649411+0000","last_clean":"2023-10-09T06:02:24.649411+0000","last_became_active":"2023-10-09T06:01:50.652208+0000","last_became_peered":"2023-10-09T06:01:50.652208+0000","last_unstale":"2023-10-09T06:02:24.649411+0000","last_undegraded":"2023-10-09T06:02:24.649411+0000","last_fullsized":"2023-10-09T06:02:24.649411+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648098+0000","last_change":"2023-10-09T06:01:51.477776+0000","last_active":"2023-10-09T06:02:24.648098+0000","last_peered":"2023-10-09T06:02:24.648098+0000","last_clean":"2023-10-09T06:02:24.648098+0000","last_became_active":"2023-10-09T06:01:51.476653+0000","last_became_peered":"2023-10-09T06:01:51.476653+0000","last_unstale":"2023-10-09T06:02:24.648098+0000","last_undegraded":"2023-10-09T06:02:24.648098+0000","last_fullsized":"2023-10-09T06:02:24.648098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647900+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:02:24.647900+0000","last_peered":"2023-10-09T06:02:24.647900+0000","last_clean":"2023-10-09T06:02:24.647900+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:02:24.647900+0000","last_undegraded":"2023-10-09T06:02:24.647900+0000","last_fullsized":"2023-10-09T06:02:24.647900+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655373+0000","last_change":"2023-10-09T06:01:51.770691+0000","last_active":"2023-10-09T06:02:24.655373+0000","last_peered":"2023-10-09T06:02:24.655373+0000","last_clean":"2023-10-09T06:02:24.655373+0000","last_became_active":"2023-10-09T06:01:51.770218+0000","last_became_peered":"2023-10-09T06:01:51.770218+0000","last_unstale":"2023-10-09T06:02:24.655373+0000","last_undegraded":"2023-10-09T06:02:24.655373+0000","last_fullsized":"2023-10-09T06:02:24.655373+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649055+0000","last_change":"2023-10-09T06:01:50.654185+0000","last_active":"2023-10-09T06:02:24.649055+0000","last_peered":"2023-10-09T06:02:24.649055+0000","last_clean":"2023-10-09T06:02:24.649055+0000","last_became_active":"2023-10-09T06:01:50.653699+0000","last_became_peered":"2023-10-09T06:01:50.653699+0000","last_unstale":"2023-10-09T06:02:24.649055+0000","last_undegraded":"2023-10-09T06:02:24.649055+0000","last_fullsized":"2023-10-09T06:02:24.649055+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651723+0000","last_change":"2023-10-09T06:01:50.678125+0000","last_active":"2023-10-09T06:02:24.651723+0000","last_peered":"2023-10-09T06:02:24.651723+0000","last_clean":"2023-10-09T06:02:24.651723+0000","last_became_active":"2023-10-09T06:01:50.677766+0000","last_became_peered":"2023-10-09T06:01:50.677766+0000","last_unstale":"2023-10-09T06:02:24.651723+0000","last_undegraded":"2023-10-09T06:02:24.651723+0000","last_fullsized":"2023-10-09T06:02:24.651723+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656948+0000","last_change":"2023-10-09T06:01:50.651500+0000","last_active":"2023-10-09T06:02:24.656948+0000","last_peered":"2023-10-09T06:02:24.656948+0000","last_clean":"2023-10-09T06:02:24.656948+0000","last_became_active":"2023-10-09T06:01:50.651172+0000","last_became_peered":"2023-10-09T06:01:50.651172+0000","last_unstale":"2023-10-09T06:02:24.656948+0000","last_undegraded":"2023-10-09T06:02:24.656948+0000","last_fullsized":"2023-10-09T06:02:24.656948+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.083506+0000","last_change":"2023-10-09T06:02:26.083506+0000","last_active":"2023-10-09T06:02:26.083506+0000","last_peered":"2023-10-09T06:02:26.083506+0000","last_clean":"2023-10-09T06:02:26.083506+0000","last_became_active":"2023-10-09T06:02:26.082579+0000","last_became_peered":"2023-10-09T06:02:26.082579+0000","last_unstale":"2023-10-09T06:02:26.083506+0000","last_undegraded":"2023-10-09T06:02:26.083506+0000","last_fullsized":"2023-10-09T06:02:26.083506+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.646302+0000","last_change":"2023-10-09T06:01:50.658448+0000","last_active":"2023-10-09T06:02:24.646302+0000","last_peered":"2023-10-09T06:02:24.646302+0000","last_clean":"2023-10-09T06:02:24.646302+0000","last_became_active":"2023-10-09T06:01:50.658007+0000","last_became_peered":"2023-10-09T06:01:50.658007+0000","last_unstale":"2023-10-09T06:02:24.646302+0000","last_undegraded":"2023-10-09T06:02:24.646302+0000","last_fullsized":"2023-10-09T06:02:24.646302+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655157+0000","last_change":"2023-10-09T06:01:51.477390+0000","last_active":"2023-10-09T06:02:24.655157+0000","last_peered":"2023-10-09T06:02:24.655157+0000","last_clean":"2023-10-09T06:02:24.655157+0000","last_became_active":"2023-10-09T06:01:51.476482+0000","last_became_peered":"2023-10-09T06:01:51.476482+0000","last_unstale":"2023-10-09T06:02:24.655157+0000","last_undegraded":"2023-10-09T06:02:24.655157+0000","last_fullsized":"2023-10-09T06:02:24.655157+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651615+0000","last_change":"2023-10-09T06:01:50.655973+0000","last_active":"2023-10-09T06:02:24.651615+0000","last_peered":"2023-10-09T06:02:24.651615+0000","last_clean":"2023-10-09T06:02:24.651615+0000","last_became_active":"2023-10-09T06:01:50.655664+0000","last_became_peered":"2023-10-09T06:01:50.655664+0000","last_unstale":"2023-10-09T06:02:24.651615+0000","last_undegraded":"2023-10-09T06:02:24.651615+0000","last_fullsized":"2023-10-09T06:02:24.651615+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645797+0000","last_change":"2023-10-09T06:01:50.674862+0000","last_active":"2023-10-09T06:02:24.645797+0000","last_peered":"2023-10-09T06:02:24.645797+0000","last_clean":"2023-10-09T06:02:24.645797+0000","last_became_active":"2023-10-09T06:01:50.674566+0000","last_became_peered":"2023-10-09T06:01:50.674566+0000","last_unstale":"2023-10-09T06:02:24.645797+0000","last_undegraded":"2023-10-09T06:02:24.645797+0000","last_fullsized":"2023-10-09T06:02:24.645797+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655375+0000","last_change":"2023-10-09T06:01:50.655797+0000","last_active":"2023-10-09T06:02:24.655375+0000","last_peered":"2023-10-09T06:02:24.655375+0000","last_clean":"2023-10-09T06:02:24.655375+0000","last_became_active":"2023-10-09T06:01:50.655032+0000","last_became_peered":"2023-10-09T06:01:50.655032+0000","last_unstale":"2023-10-09T06:02:24.655375+0000","last_undegraded":"2023-10-09T06:02:24.655375+0000","last_fullsized":"2023-10-09T06:02:24.655375+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651985+0000","last_change":"2023-10-09T06:01:51.473771+0000","last_active":"2023-10-09T06:02:24.651985+0000","last_peered":"2023-10-09T06:02:24.651985+0000","last_clean":"2023-10-09T06:02:24.651985+0000","last_became_active":"2023-10-09T06:01:51.473476+0000","last_became_peered":"2023-10-09T06:01:51.473476+0000","last_unstale":"2023-10-09T06:02:24.651985+0000","last_undegraded":"2023-10-09T06:02:24.651985+0000","last_fullsized":"2023-10-09T06:02:24.651985+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647784+0000","last_change":"2023-10-09T06:01:50.657344+0000","last_active":"2023-10-09T06:02:24.647784+0000","last_peered":"2023-10-09T06:02:24.647784+0000","last_clean":"2023-10-09T06:02:24.647784+0000","last_became_active":"2023-10-09T06:01:50.657015+0000","last_became_peered":"2023-10-09T06:01:50.657015+0000","last_unstale":"2023-10-09T06:02:24.647784+0000","last_undegraded":"2023-10-09T06:02:24.647784+0000","last_fullsized":"2023-10-09T06:02:24.647784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430265,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.85799999999999998,"15min":0.878},"min":{"1min":0.60699999999999998,"5min":0.371,"15min":0.371},"max":{"1min":1.274,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.234},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.93000000000000005,"15min":0.93400000000000005},"min":{"1min":0.70399999999999996,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2549999999999999,"5min":1.452,"15min":1.452},"last":1.127}]},{"osd":1,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.88700000000000001,"15min":0.89200000000000002},"min":{"1min":0.64800000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3080000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.94399999999999995,"15min":0.97399999999999998},"min":{"1min":0.69399999999999995,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.216,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.169}]},{"osd":2,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.88700000000000001,"15min":0.90000000000000002},"min":{"1min":0.68000000000000005,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.90700000000000003,"15min":0.90300000000000002},"min":{"1min":0.69499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1779999999999999,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":0.76500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93899999999999995,"15min":0.95999999999999996},"min":{"1min":0.65600000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.24,"5min":1.335,"15min":1.335},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.997,"5min":0.94799999999999995,"15min":0.95399999999999996},"min":{"1min":0.75800000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.248,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":0.92700000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.93999999999999995,"15min":0.95499999999999996},"min":{"1min":0.65600000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3149999999999999,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.90500000000000003,"15min":0.90300000000000002},"min":{"1min":0.53600000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2,"5min":1.51,"15min":1.51},"last":1.0209999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96699999999999997,"15min":0.98099999999999998},"min":{"1min":0.60099999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.359,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":1.05},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.03,"15min":1.022},"min":{"1min":0.874,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3939999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.278}]},{"osd":6,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.0660000000000001,"15min":1.0569999999999999},"min":{"1min":0.88900000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.4399999999999999,"5min":1.508,"15min":1.508},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.044,"15min":1.044},"min":{"1min":0.85099999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.2989999999999999,"5min":2.0019999999999998,"15min":2.0019999999999998},"last":1.1459999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626497,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.873,"15min":0.88300000000000001},"min":{"1min":0.51800000000000002,"5min":0.439,"15min":0.439},"max":{"1min":1.1299999999999999,"5min":1.294,"15min":1.294},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.873,"15min":0.88900000000000001},"min":{"1min":0.54800000000000004,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3160000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.99199999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83799999999999997,"15min":0.83299999999999996},"min":{"1min":0.52700000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.3680000000000001,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.86899999999999999,"15min":0.83099999999999996},"min":{"1min":0.59199999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.137,"5min":1.278,"15min":1.278},"last":1.0289999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90300000000000002,"15min":0.878},"min":{"1min":0.67300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.2150000000000001,"5min":1.423,"15min":1.423},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.93799999999999994,"15min":0.96499999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.367,"5min":1.786,"15min":1.786},"last":0.83099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.88500000000000001,"15min":0.86799999999999999},"min":{"1min":0.63,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.03,"5min":0.99299999999999999,"15min":0.95699999999999996},"min":{"1min":0.68999999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3400000000000001,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":1.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.93000000000000005,"15min":0.89300000000000002},"min":{"1min":0.77100000000000002,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.504,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.083,"5min":0.95199999999999996,"15min":0.90100000000000002},"min":{"1min":0.81299999999999994,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.3149999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.208}]},{"osd":5,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.97999999999999998,"15min":0.878},"min":{"1min":0.88400000000000001,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2749999999999999,"5min":1.708,"15min":1.708},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":0.997,"15min":0.93200000000000005},"min":{"1min":0.70599999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.4099999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.2949999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:03:00 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.075,"15min":1.0349999999999999},"min":{"1min":0.87,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4410000000000001,"5min":1.629,"15min":1.629},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.087,"15min":1.052},"min":{"1min":0.73099999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.4550000000000001,"5min":1.47,"15min":1.47},"last":1.1559999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574958,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":269625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.70999999999999996,"15min":0.628},"min":{"1min":0.375,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":1.1519999999999999,"5min":1.3600000000000001,"15min":1.5369999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.751,"5min":0.69699999999999995,"15min":0.626},"min":{"1min":0.51400000000000001,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":0.97799999999999998,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.76200000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.75,"15min":0.67300000000000004},"min":{"1min":0.47199999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.96899999999999997,"5min":1.464,"15min":1.464},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.745,"5min":0.72699999999999998,"15min":0.66800000000000004},"min":{"1min":0.49399999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.012,"5min":1.234,"15min":1.234},"last":0.622}]},{"osd":3,"last update":"Mon Oct 9 06:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.78100000000000003,"15min":0.80100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.423,"5min":1.423,"15min":1.5820000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.746,"15min":0.74199999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.125,"5min":1.125,"15min":1.3480000000000001},"last":0.68600000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:02:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.89000000000000001,"15min":0.89100000000000001},"min":{"1min":0.62,"5min":0.56799999999999995,"15min":0.54700000000000004},"max":{"1min":1.0940000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.087},{"interface":"front","average":{"1min":1.03,"5min":1.0109999999999999,"15min":0.99099999999999999},"min":{"1min":0.68200000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.3300000000000001,"5min":1.575,"15min":1.6419999999999999},"last":0.86399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.753,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.2829999999999999,"5min":1.294,"15min":1.294},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.97999999999999998,"15min":0.96499999999999997},"min":{"1min":0.81999999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2909999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.0209999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:02:17 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.093,"15min":1.095},"min":{"1min":0.84799999999999998,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0329999999999999,"15min":1.05},"min":{"1min":0.80100000000000005,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.98799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0840000000000001,"15min":1.1120000000000001},"min":{"1min":0.70999999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.4430000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1739999999999999,"15min":1.212},"min":{"1min":0.71899999999999997,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.54,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.044}]}]},{"osd":0,"up_from":8,"seq":34359738486,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.63,"15min":0.57099999999999995},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":0.96499999999999997,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.872},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.73199999999999998,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.167,"5min":1.1930000000000001,"15min":1.22},"last":0.65900000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.73099999999999998,"15min":0.67400000000000004},"min":{"1min":0.50600000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88400000000000001,"5min":1.2090000000000001,"15min":1.3089999999999999},"last":0.746},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.72199999999999998,"15min":0.64800000000000002},"min":{"1min":0.309,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.90900000000000003,"5min":1.413,"15min":1.413},"last":0.91400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.78300000000000003,"15min":0.79900000000000004},"min":{"1min":0.435,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.1120000000000001,"5min":1.1519999999999999,"15min":1.387},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.79900000000000004,"15min":0.79300000000000004},"min":{"1min":0.39200000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.054,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.83499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.88,"15min":0.84999999999999998},"min":{"1min":0.67900000000000005,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.206,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":1.028},{"interface":"front","average":{"1min":1.498,"5min":1.0800000000000001,"15min":1.0209999999999999},"min":{"1min":0.75700000000000001,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":9.0280000000000005,"5min":9.0280000000000005,"15min":9.0280000000000005},"last":1.181}]},{"osd":5,"last update":"Mon Oct 9 06:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.995,"15min":0.99099999999999999},"min":{"1min":0.59999999999999998,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4139999999999999,"5min":1.968,"15min":1.968},"last":1.24},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.016},"min":{"1min":0.66300000000000003,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.403,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.21}]},{"osd":6,"last update":"Mon Oct 9 06:02:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.0549999999999999,"15min":1.1160000000000001},"min":{"1min":0.74299999999999999,"5min":0.505,"15min":0.505},"max":{"1min":1.3360000000000001,"5min":2.0409999999999999,"15min":2.0409999999999999},"last":1.159},{"interface":"front","average":{"1min":1.409,"5min":1.1000000000000001,"15min":1.083},"min":{"1min":0.84399999999999997,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":6.7290000000000001,"5min":6.7290000000000001,"15min":6.7290000000000001},"last":1.387}]},{"osd":7,"last update":"Mon Oct 9 06:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.0189999999999999,"15min":1.044},"min":{"1min":0.627,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.472,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.335,"5min":1.1200000000000001,"15min":1.0880000000000001},"min":{"1min":0.74099999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":5.1820000000000004,"5min":5.1820000000000004,"15min":5.1820000000000004},"last":1.3440000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411429,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.67500000000000004,"15min":0.63300000000000001},"min":{"1min":0.31900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":1.171,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.63200000000000001},"min":{"1min":0.41899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.95099999999999996,"5min":2.2719999999999998,"15min":2.2719999999999998},"last":0.58099999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.70699999999999996,"15min":0.64600000000000002},"min":{"1min":0.45000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.03,"5min":1.496,"15min":1.496},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.746,"15min":0.69599999999999995},"min":{"1min":0.51000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.022,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.74299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.77800000000000002,"15min":0.76700000000000002},"min":{"1min":0.40200000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.05,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79900000000000004,"15min":0.78700000000000003},"min":{"1min":0.58199999999999996,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.165,"5min":1.546,"15min":1.546},"last":0.998}]},{"osd":4,"last update":"Mon Oct 9 06:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.95599999999999996,"15min":0.93100000000000005},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.9810000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.006,"15min":1.032},"min":{"1min":0.58799999999999997,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.92800000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.95899999999999996,"15min":0.98199999999999998},"min":{"1min":0.64400000000000002,"5min":0.505,"15min":0.505},"max":{"1min":1.4199999999999999,"5min":1.734,"15min":1.734},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.022,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":2.2669999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":1.0669999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:02:23 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.03,"15min":1.048},"min":{"1min":0.73799999999999999,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.45,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.155,"5min":1.0800000000000001,"15min":1.091},"min":{"1min":0.78100000000000003,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.4830000000000001,"5min":1.8839999999999999,"15min":1.8839999999999999},"last":1.1000000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.054,"15min":1.04},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.8740000000000001,"5min":2.1160000000000001,"15min":2.1160000000000001},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.0680000000000001,"15min":1.056},"min":{"1min":0.628,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.21,"5min":2.21,"15min":2.21},"last":1.2050000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182491,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.33300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.0740000000000001,"5min":1.232,"15min":1.232},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.69899999999999995,"15min":0.72399999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81799999999999995,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.65600000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.71899999999999997,"15min":0.77600000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.1910000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.68999999999999995,"15min":0.76300000000000001},"min":{"1min":0.40000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.036,"5min":1.036,"15min":1.4319999999999999},"last":0.70199999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.73499999999999999,"15min":0.75800000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.0029999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.76800000000000002,"15min":0.79300000000000004},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.429,"5min":1.754,"15min":1.754},"last":0.88}]},{"osd":4,"last update":"Mon Oct 9 06:02:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.93899999999999995,"15min":0.90600000000000003},"min":{"1min":0.68600000000000005,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3380000000000001,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97499999999999998,"15min":0.93100000000000005},"min":{"1min":0.76800000000000002,"5min":0.622,"15min":0.622},"max":{"1min":1.2010000000000001,"5min":1.496,"15min":1.496},"last":0.96099999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:02:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.006,"15min":1.036},"min":{"1min":0.72199999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.325,"5min":1.556,"15min":1.556},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.84399999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2529999999999999,"5min":1.651,"15min":1.651},"last":1.095}]},{"osd":6,"last update":"Mon Oct 9 06:03:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0109999999999999,"15min":1.093},"min":{"1min":0.69899999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.194},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.046,"15min":1.1100000000000001},"min":{"1min":0.63300000000000001,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.2509999999999999,"5min":1.609,"15min":1.609},"last":1.0249999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:02:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0860000000000001,"15min":1.1200000000000001},"min":{"1min":0.75,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.111,"15min":1.1379999999999999},"min":{"1min":0.747,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.361,"5min":1.665,"15min":1.665},"last":1.236}]}]},{"osd":4,"up_from":30,"seq":128849018963,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86699999999999999,"15min":0.84699999999999998},"min":{"1min":0.57399999999999995,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.2809999999999999,"5min":1.8049999999999999,"15min":1.8049999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.016,"5min":0.91000000000000003,"15min":0.84399999999999997},"min":{"1min":0.753,"5min":0.52000000000000002,"15min":0.41999999999999998},"max":{"1min":1.419,"5min":1.927,"15min":1.927},"last":1}]},{"osd":1,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.88300000000000001,"15min":0.81499999999999995},"min":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3200000000000001,"5min":2.3410000000000002,"15min":2.3410000000000002},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.95499999999999996,"15min":0.88700000000000001},"min":{"1min":0.58299999999999996,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.524,"5min":3.3690000000000002,"15min":3.3690000000000002},"last":1.145}]},{"osd":2,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.94799999999999995,"15min":0.89600000000000002},"min":{"1min":0.61099999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.93400000000000005,"15min":0.88800000000000001},"min":{"1min":0.64100000000000001,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.349,"5min":2.1150000000000002,"15min":2.1150000000000002},"last":1.1779999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.96399999999999997,"15min":0.88800000000000001},"min":{"1min":0.54900000000000004,"5min":0.497,"15min":0.497},"max":{"1min":1.4870000000000001,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":1.105},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.98299999999999998,"15min":0.89300000000000002},"min":{"1min":0.68799999999999994,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.238,"5min":2.5619999999999998,"15min":2.5619999999999998},"last":0.96799999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.877,"15min":0.77700000000000002},"min":{"1min":0.61799999999999999,"5min":0.372,"15min":0.372},"max":{"1min":1.869,"5min":2.79,"15min":2.79},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.90900000000000003,"15min":0.79300000000000004},"min":{"1min":0.75800000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":1.2529999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.998,"15min":0.97099999999999997},"min":{"1min":0.71299999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3009999999999999,"5min":1.891,"15min":1.891},"last":1.206},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.022,"15min":0.99099999999999999},"min":{"1min":0.80500000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.248,"5min":2.915,"15min":2.915},"last":1.127}]},{"osd":7,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.085,"15min":1.077},"min":{"1min":0.77800000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3740000000000001,"5min":2.056,"15min":2.056},"last":1.298},{"interface":"front","average":{"1min":1.103,"5min":1.0920000000000001,"15min":1.081},"min":{"1min":0.83399999999999996,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.3560000000000001,"5min":2.629,"15min":2.629},"last":1.2749999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822730,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.872,"15min":0.86499999999999999},"min":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3420000000000001,"5min":1.492,"15min":1.492},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89000000000000001,"15min":0.874},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.77200000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.90700000000000003,"15min":0.88900000000000001},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3049999999999999,"5min":1.377,"15min":1.377},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.88600000000000001,"15min":0.89400000000000002},"min":{"1min":0.57199999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1779999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.0069999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.89000000000000001,"15min":0.89400000000000002},"min":{"1min":0.66600000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.93999999999999995},"min":{"1min":0.72999999999999998,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3859999999999999,"5min":1.609,"15min":1.609},"last":1.056}]},{"osd":3,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.91100000000000003,"15min":0.88700000000000001},"min":{"1min":0.71799999999999997,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.22,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.119},{"interface":"front","average":{"1min":1.012,"5min":0.98799999999999999,"15min":0.97399999999999998},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.349,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.87,"15min":0.82199999999999995},"min":{"1min":0.65000000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.3080000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.89400000000000002,"15min":0.80800000000000005},"min":{"1min":0.63200000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.536,"5min":1.718,"15min":1.718},"last":1.04}]},{"osd":6,"last update":"Mon Oct 9 06:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.97399999999999998,"15min":0.96199999999999997},"min":{"1min":0.81200000000000006,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.365,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.26},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0149999999999999,"15min":1.0289999999999999},"min":{"1min":0.63800000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.385,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.0820000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0649999999999999,"15min":1.079},"min":{"1min":0.56200000000000006,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3899999999999999,"5min":1.718,"15min":1.718},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.119,"5min":1.101,"15min":1.1160000000000001},"min":{"1min":0.72299999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.3939999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.145}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:03:25.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:03:25.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:25 smithi125 bash[27513]: cluster 2023-10-09T06:03:24.252056+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:25 smithi125 bash[27513]: audit 2023-10-09T06:03:24.408803+0000 mgr.y (mgr.14430) 154 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:25 smithi125 bash[33919]: cluster 2023-10-09T06:03:24.252056+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:25 smithi125 bash[33919]: audit 2023-10-09T06:03:24.408803+0000 mgr.y (mgr.14430) 154 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:25 smithi155 bash[27817]: cluster 2023-10-09T06:03:24.252056+0000 mgr.y (mgr.14430) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:25 smithi155 bash[27817]: audit 2023-10-09T06:03:24.408803+0000 mgr.y (mgr.14430) 154 : audit [DBG] from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:27.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:27 smithi125 bash[27513]: cluster 2023-10-09T06:03:26.253114+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:27 smithi125 bash[33919]: cluster 2023-10-09T06:03:26.253114+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:27 smithi155 bash[27817]: cluster 2023-10-09T06:03:26.253114+0000 mgr.y (mgr.14430) 155 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:28] "GET /metrics HTTP/1.1" 200 191543 "" "Prometheus/2.33.4" 2023-10-09T06:03:29.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:29 smithi125 bash[27513]: cluster 2023-10-09T06:03:28.253743+0000 mgr.y (mgr.14430) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:29 smithi125 bash[33919]: cluster 2023-10-09T06:03:28.253743+0000 mgr.y (mgr.14430) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:29 smithi155 bash[27817]: cluster 2023-10-09T06:03:28.253743+0000 mgr.y (mgr.14430) 156 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:30.027 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:31.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:31 smithi155 bash[27817]: cluster 2023-10-09T06:03:30.254768+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:31.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:31 smithi125 bash[27513]: cluster 2023-10-09T06:03:30.254768+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:31.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:31 smithi125 bash[33919]: cluster 2023-10-09T06:03:30.254768+0000 mgr.y (mgr.14430) 157 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:32.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:31 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:33.348 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:03:33.351 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:03:33.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:33 smithi125 bash[33919]: cluster 2023-10-09T06:03:32.255324+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:33 smithi125 bash[27513]: cluster 2023-10-09T06:03:32.255324+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:33 smithi155 bash[27817]: cluster 2023-10-09T06:03:32.255324+0000 mgr.y (mgr.14430) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:34.118 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":138,"stamp":"2023-10-09T06:03:32.254997+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381740,"kb_used_data":4972,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579476,"statfs":{"total":767960285184,"available":765521383424,"internally_reserved":0,"allocated":5091328,"data_stored":2164863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004541"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647513+0000","last_change":"2023-10-09T06:01:51.474967+0000","last_active":"2023-10-09T06:02:24.647513+0000","last_peered":"2023-10-09T06:02:24.647513+0000","last_clean":"2023-10-09T06:02:24.647513+0000","last_became_active":"2023-10-09T06:01:51.473515+0000","last_became_peered":"2023-10-09T06:01:51.473515+0000","last_unstale":"2023-10-09T06:02:24.647513+0000","last_undegraded":"2023-10-09T06:02:24.647513+0000","last_fullsized":"2023-10-09T06:02:24.647513+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.646361+0000","last_change":"2023-10-09T06:01:50.649268+0000","last_active":"2023-10-09T06:02:24.646361+0000","last_peered":"2023-10-09T06:02:24.646361+0000","last_clean":"2023-10-09T06:02:24.646361+0000","last_became_active":"2023-10-09T06:01:50.648924+0000","last_became_peered":"2023-10-09T06:01:50.648924+0000","last_unstale":"2023-10-09T06:02:24.646361+0000","last_undegraded":"2023-10-09T06:02:24.646361+0000","last_fullsized":"2023-10-09T06:02:24.646361+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651639+0000","last_change":"2023-10-09T06:01:50.672522+0000","last_active":"2023-10-09T06:02:24.651639+0000","last_peered":"2023-10-09T06:02:24.651639+0000","last_clean":"2023-10-09T06:02:24.651639+0000","last_became_active":"2023-10-09T06:01:50.672103+0000","last_became_peered":"2023-10-09T06:01:50.672103+0000","last_unstale":"2023-10-09T06:02:24.651639+0000","last_undegraded":"2023-10-09T06:02:24.651639+0000","last_fullsized":"2023-10-09T06:02:24.651639+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652145+0000","last_change":"2023-10-09T06:01:50.649515+0000","last_active":"2023-10-09T06:02:24.652145+0000","last_peered":"2023-10-09T06:02:24.652145+0000","last_clean":"2023-10-09T06:02:24.652145+0000","last_became_active":"2023-10-09T06:01:50.649150+0000","last_became_peered":"2023-10-09T06:01:50.649150+0000","last_unstale":"2023-10-09T06:02:24.652145+0000","last_undegraded":"2023-10-09T06:02:24.652145+0000","last_fullsized":"2023-10-09T06:02:24.652145+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647973+0000","last_change":"2023-10-09T06:01:50.673776+0000","last_active":"2023-10-09T06:02:24.647973+0000","last_peered":"2023-10-09T06:02:24.647973+0000","last_clean":"2023-10-09T06:02:24.647973+0000","last_became_active":"2023-10-09T06:01:50.673445+0000","last_became_peered":"2023-10-09T06:01:50.673445+0000","last_unstale":"2023-10-09T06:02:24.647973+0000","last_undegraded":"2023-10-09T06:02:24.647973+0000","last_fullsized":"2023-10-09T06:02:24.647973+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657125+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:02:24.657125+0000","last_peered":"2023-10-09T06:02:24.657125+0000","last_clean":"2023-10-09T06:02:24.657125+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:02:24.657125+0000","last_undegraded":"2023-10-09T06:02:24.657125+0000","last_fullsized":"2023-10-09T06:02:24.657125+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648152+0000","last_change":"2023-10-09T06:01:51.478485+0000","last_active":"2023-10-09T06:02:24.648152+0000","last_peered":"2023-10-09T06:02:24.648152+0000","last_clean":"2023-10-09T06:02:24.648152+0000","last_became_active":"2023-10-09T06:01:51.477533+0000","last_became_peered":"2023-10-09T06:01:51.477533+0000","last_unstale":"2023-10-09T06:02:24.648152+0000","last_undegraded":"2023-10-09T06:02:24.648152+0000","last_fullsized":"2023-10-09T06:02:24.648152+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657000+0000","last_change":"2023-10-09T06:01:50.652132+0000","last_active":"2023-10-09T06:02:24.657000+0000","last_peered":"2023-10-09T06:02:24.657000+0000","last_clean":"2023-10-09T06:02:24.657000+0000","last_became_active":"2023-10-09T06:01:50.651802+0000","last_became_peered":"2023-10-09T06:01:50.651802+0000","last_unstale":"2023-10-09T06:02:24.657000+0000","last_undegraded":"2023-10-09T06:02:24.657000+0000","last_fullsized":"2023-10-09T06:02:24.657000+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.536637+0000","last_change":"2023-10-09T06:02:26.536637+0000","last_active":"2023-10-09T06:02:26.536637+0000","last_peered":"2023-10-09T06:02:26.536637+0000","last_clean":"2023-10-09T06:02:26.536637+0000","last_became_active":"2023-10-09T06:02:26.535915+0000","last_became_peered":"2023-10-09T06:02:26.535915+0000","last_unstale":"2023-10-09T06:02:26.536637+0000","last_undegraded":"2023-10-09T06:02:26.536637+0000","last_fullsized":"2023-10-09T06:02:26.536637+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645719+0000","last_change":"2023-10-09T06:01:50.650579+0000","last_active":"2023-10-09T06:02:24.645719+0000","last_peered":"2023-10-09T06:02:24.645719+0000","last_clean":"2023-10-09T06:02:24.645719+0000","last_became_active":"2023-10-09T06:01:50.650246+0000","last_became_peered":"2023-10-09T06:01:50.650246+0000","last_unstale":"2023-10-09T06:02:24.645719+0000","last_undegraded":"2023-10-09T06:02:24.645719+0000","last_fullsized":"2023-10-09T06:02:24.645719+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647940+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:02:24.647940+0000","last_peered":"2023-10-09T06:02:24.647940+0000","last_clean":"2023-10-09T06:02:24.647940+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:02:24.647940+0000","last_undegraded":"2023-10-09T06:02:24.647940+0000","last_fullsized":"2023-10-09T06:02:24.647940+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645917+0000","last_change":"2023-10-09T06:01:50.648001+0000","last_active":"2023-10-09T06:02:24.645917+0000","last_peered":"2023-10-09T06:02:24.645917+0000","last_clean":"2023-10-09T06:02:24.645917+0000","last_became_active":"2023-10-09T06:01:50.647686+0000","last_became_peered":"2023-10-09T06:01:50.647686+0000","last_unstale":"2023-10-09T06:02:24.645917+0000","last_undegraded":"2023-10-09T06:02:24.645917+0000","last_fullsized":"2023-10-09T06:02:24.645917+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655290+0000","last_change":"2023-10-09T05:59:52.469538+0000","last_active":"2023-10-09T06:02:24.655290+0000","last_peered":"2023-10-09T06:02:24.655290+0000","last_clean":"2023-10-09T06:02:24.655290+0000","last_became_active":"2023-10-09T05:59:52.469011+0000","last_became_peered":"2023-10-09T05:59:52.469011+0000","last_unstale":"2023-10-09T06:02:24.655290+0000","last_undegraded":"2023-10-09T06:02:24.655290+0000","last_fullsized":"2023-10-09T06:02:24.655290+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651405+0000","last_change":"2023-10-09T06:01:50.678265+0000","last_active":"2023-10-09T06:02:24.651405+0000","last_peered":"2023-10-09T06:02:24.651405+0000","last_clean":"2023-10-09T06:02:24.651405+0000","last_became_active":"2023-10-09T06:01:50.677885+0000","last_became_peered":"2023-10-09T06:01:50.677885+0000","last_unstale":"2023-10-09T06:02:24.651405+0000","last_undegraded":"2023-10-09T06:02:24.651405+0000","last_fullsized":"2023-10-09T06:02:24.651405+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655952+0000","last_change":"2023-10-09T05:59:52.779228+0000","last_active":"2023-10-09T06:02:24.655952+0000","last_peered":"2023-10-09T06:02:24.655952+0000","last_clean":"2023-10-09T06:02:24.655952+0000","last_became_active":"2023-10-09T05:59:52.777994+0000","last_became_peered":"2023-10-09T05:59:52.777994+0000","last_unstale":"2023-10-09T06:02:24.655952+0000","last_undegraded":"2023-10-09T06:02:24.655952+0000","last_fullsized":"2023-10-09T06:02:24.655952+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652312+0000","last_change":"2023-10-09T06:01:50.656699+0000","last_active":"2023-10-09T06:02:24.652312+0000","last_peered":"2023-10-09T06:02:24.652312+0000","last_clean":"2023-10-09T06:02:24.652312+0000","last_became_active":"2023-10-09T06:01:50.656372+0000","last_became_peered":"2023-10-09T06:01:50.656372+0000","last_unstale":"2023-10-09T06:02:24.652312+0000","last_undegraded":"2023-10-09T06:02:24.652312+0000","last_fullsized":"2023-10-09T06:02:24.652312+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645134+0000","last_change":"2023-10-09T06:01:50.678079+0000","last_active":"2023-10-09T06:02:24.645134+0000","last_peered":"2023-10-09T06:02:24.645134+0000","last_clean":"2023-10-09T06:02:24.645134+0000","last_became_active":"2023-10-09T06:01:50.677755+0000","last_became_peered":"2023-10-09T06:01:50.677755+0000","last_unstale":"2023-10-09T06:02:24.645134+0000","last_undegraded":"2023-10-09T06:02:24.645134+0000","last_fullsized":"2023-10-09T06:02:24.645134+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656309+0000","last_change":"2023-10-09T06:01:50.655267+0000","last_active":"2023-10-09T06:02:24.656309+0000","last_peered":"2023-10-09T06:02:24.656309+0000","last_clean":"2023-10-09T06:02:24.656309+0000","last_became_active":"2023-10-09T06:01:50.654934+0000","last_became_peered":"2023-10-09T06:01:50.654934+0000","last_unstale":"2023-10-09T06:02:24.656309+0000","last_undegraded":"2023-10-09T06:02:24.656309+0000","last_fullsized":"2023-10-09T06:02:24.656309+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649109+0000","last_change":"2023-10-09T05:59:52.778611+0000","last_active":"2023-10-09T06:02:24.649109+0000","last_peered":"2023-10-09T06:02:24.649109+0000","last_clean":"2023-10-09T06:02:24.649109+0000","last_became_active":"2023-10-09T05:59:52.777893+0000","last_became_peered":"2023-10-09T05:59:52.777893+0000","last_unstale":"2023-10-09T06:02:24.649109+0000","last_undegraded":"2023-10-09T06:02:24.649109+0000","last_fullsized":"2023-10-09T06:02:24.649109+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645925+0000","last_change":"2023-10-09T06:01:51.475796+0000","last_active":"2023-10-09T06:02:24.645925+0000","last_peered":"2023-10-09T06:02:24.645925+0000","last_clean":"2023-10-09T06:02:24.645925+0000","last_became_active":"2023-10-09T06:01:51.475387+0000","last_became_peered":"2023-10-09T06:01:51.475387+0000","last_unstale":"2023-10-09T06:02:24.645925+0000","last_undegraded":"2023-10-09T06:02:24.645925+0000","last_fullsized":"2023-10-09T06:02:24.645925+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645258+0000","last_change":"2023-10-09T06:01:50.658288+0000","last_active":"2023-10-09T06:02:24.645258+0000","last_peered":"2023-10-09T06:02:24.645258+0000","last_clean":"2023-10-09T06:02:24.645258+0000","last_became_active":"2023-10-09T06:01:50.657871+0000","last_became_peered":"2023-10-09T06:01:50.657871+0000","last_unstale":"2023-10-09T06:02:24.645258+0000","last_undegraded":"2023-10-09T06:02:24.645258+0000","last_fullsized":"2023-10-09T06:02:24.645258+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648108+0000","last_change":"2023-10-09T05:59:52.583913+0000","last_active":"2023-10-09T06:02:24.648108+0000","last_peered":"2023-10-09T06:02:24.648108+0000","last_clean":"2023-10-09T06:02:24.648108+0000","last_became_active":"2023-10-09T05:59:52.583298+0000","last_became_peered":"2023-10-09T05:59:52.583298+0000","last_unstale":"2023-10-09T06:02:24.648108+0000","last_undegraded":"2023-10-09T06:02:24.648108+0000","last_fullsized":"2023-10-09T06:02:24.648108+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654996+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:02:24.654996+0000","last_peered":"2023-10-09T06:02:24.654996+0000","last_clean":"2023-10-09T06:02:24.654996+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:02:24.654996+0000","last_undegraded":"2023-10-09T06:02:24.654996+0000","last_fullsized":"2023-10-09T06:02:24.654996+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651789+0000","last_change":"2023-10-09T06:01:50.650581+0000","last_active":"2023-10-09T06:02:24.651789+0000","last_peered":"2023-10-09T06:02:24.651789+0000","last_clean":"2023-10-09T06:02:24.651789+0000","last_became_active":"2023-10-09T06:01:50.650078+0000","last_became_peered":"2023-10-09T06:01:50.650078+0000","last_unstale":"2023-10-09T06:02:24.651789+0000","last_undegraded":"2023-10-09T06:02:24.651789+0000","last_fullsized":"2023-10-09T06:02:24.651789+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648015+0000","last_change":"2023-10-09T05:58:52.535269+0000","last_active":"2023-10-09T06:02:24.648015+0000","last_peered":"2023-10-09T06:02:24.648015+0000","last_clean":"2023-10-09T06:02:24.648015+0000","last_became_active":"2023-10-09T05:58:52.534584+0000","last_became_peered":"2023-10-09T05:58:52.534584+0000","last_unstale":"2023-10-09T06:02:24.648015+0000","last_undegraded":"2023-10-09T06:02:24.648015+0000","last_fullsized":"2023-10-09T06:02:24.648015+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656797+0000","last_change":"2023-10-09T05:59:52.469599+0000","last_active":"2023-10-09T06:02:24.656797+0000","last_peered":"2023-10-09T06:02:24.656797+0000","last_clean":"2023-10-09T06:02:24.656797+0000","last_became_active":"2023-10-09T05:59:52.469155+0000","last_became_peered":"2023-10-09T05:59:52.469155+0000","last_unstale":"2023-10-09T06:02:24.656797+0000","last_undegraded":"2023-10-09T06:02:24.656797+0000","last_fullsized":"2023-10-09T06:02:24.656797+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655999+0000","last_change":"2023-10-09T06:01:51.771158+0000","last_active":"2023-10-09T06:02:24.655999+0000","last_peered":"2023-10-09T06:02:24.655999+0000","last_clean":"2023-10-09T06:02:24.655999+0000","last_became_active":"2023-10-09T06:01:51.770747+0000","last_became_peered":"2023-10-09T06:01:51.770747+0000","last_unstale":"2023-10-09T06:02:24.655999+0000","last_undegraded":"2023-10-09T06:02:24.655999+0000","last_fullsized":"2023-10-09T06:02:24.655999+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656105+0000","last_change":"2023-10-09T06:01:50.653085+0000","last_active":"2023-10-09T06:02:24.656105+0000","last_peered":"2023-10-09T06:02:24.656105+0000","last_clean":"2023-10-09T06:02:24.656105+0000","last_became_active":"2023-10-09T06:01:50.652800+0000","last_became_peered":"2023-10-09T06:01:50.652800+0000","last_unstale":"2023-10-09T06:02:24.656105+0000","last_undegraded":"2023-10-09T06:02:24.656105+0000","last_fullsized":"2023-10-09T06:02:24.656105+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.653991+0000","last_change":"2023-10-09T05:59:52.778084+0000","last_active":"2023-10-09T06:02:24.653991+0000","last_peered":"2023-10-09T06:02:24.653991+0000","last_clean":"2023-10-09T06:02:24.653991+0000","last_became_active":"2023-10-09T05:59:52.777374+0000","last_became_peered":"2023-10-09T05:59:52.777374+0000","last_unstale":"2023-10-09T06:02:24.653991+0000","last_undegraded":"2023-10-09T06:02:24.653991+0000","last_fullsized":"2023-10-09T06:02:24.653991+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656223+0000","last_change":"2023-10-09T06:01:51.770156+0000","last_active":"2023-10-09T06:02:24.656223+0000","last_peered":"2023-10-09T06:02:24.656223+0000","last_clean":"2023-10-09T06:02:24.656223+0000","last_became_active":"2023-10-09T06:01:51.769850+0000","last_became_peered":"2023-10-09T06:01:51.769850+0000","last_unstale":"2023-10-09T06:02:24.656223+0000","last_undegraded":"2023-10-09T06:02:24.656223+0000","last_fullsized":"2023-10-09T06:02:24.656223+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648030+0000","last_change":"2023-10-09T06:01:50.651949+0000","last_active":"2023-10-09T06:02:24.648030+0000","last_peered":"2023-10-09T06:02:24.648030+0000","last_clean":"2023-10-09T06:02:24.648030+0000","last_became_active":"2023-10-09T06:01:50.651000+0000","last_became_peered":"2023-10-09T06:01:50.651000+0000","last_unstale":"2023-10-09T06:02:24.648030+0000","last_undegraded":"2023-10-09T06:02:24.648030+0000","last_fullsized":"2023-10-09T06:02:24.648030+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656838+0000","last_change":"2023-10-09T05:59:52.467577+0000","last_active":"2023-10-09T06:02:24.656838+0000","last_peered":"2023-10-09T06:02:24.656838+0000","last_clean":"2023-10-09T06:02:24.656838+0000","last_became_active":"2023-10-09T05:59:52.467143+0000","last_became_peered":"2023-10-09T05:59:52.467143+0000","last_unstale":"2023-10-09T06:02:24.656838+0000","last_undegraded":"2023-10-09T06:02:24.656838+0000","last_fullsized":"2023-10-09T06:02:24.656838+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655405+0000","last_change":"2023-10-09T06:01:51.770429+0000","last_active":"2023-10-09T06:02:24.655405+0000","last_peered":"2023-10-09T06:02:24.655405+0000","last_clean":"2023-10-09T06:02:24.655405+0000","last_became_active":"2023-10-09T06:01:51.769840+0000","last_became_peered":"2023-10-09T06:01:51.769840+0000","last_unstale":"2023-10-09T06:02:24.655405+0000","last_undegraded":"2023-10-09T06:02:24.655405+0000","last_fullsized":"2023-10-09T06:02:24.655405+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.657062+0000","last_change":"2023-10-09T06:01:50.648057+0000","last_active":"2023-10-09T06:02:24.657062+0000","last_peered":"2023-10-09T06:02:24.657062+0000","last_clean":"2023-10-09T06:02:24.657062+0000","last_became_active":"2023-10-09T06:01:50.647717+0000","last_became_peered":"2023-10-09T06:01:50.647717+0000","last_unstale":"2023-10-09T06:02:24.657062+0000","last_undegraded":"2023-10-09T06:02:24.657062+0000","last_fullsized":"2023-10-09T06:02:24.657062+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656133+0000","last_change":"2023-10-09T05:59:52.583128+0000","last_active":"2023-10-09T06:02:24.656133+0000","last_peered":"2023-10-09T06:02:24.656133+0000","last_clean":"2023-10-09T06:02:24.656133+0000","last_became_active":"2023-10-09T05:59:52.582692+0000","last_became_peered":"2023-10-09T05:59:52.582692+0000","last_unstale":"2023-10-09T06:02:24.656133+0000","last_undegraded":"2023-10-09T06:02:24.656133+0000","last_fullsized":"2023-10-09T06:02:24.656133+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656049+0000","last_change":"2023-10-09T06:01:51.768939+0000","last_active":"2023-10-09T06:02:24.656049+0000","last_peered":"2023-10-09T06:02:24.656049+0000","last_clean":"2023-10-09T06:02:24.656049+0000","last_became_active":"2023-10-09T06:01:51.768522+0000","last_became_peered":"2023-10-09T06:01:51.768522+0000","last_unstale":"2023-10-09T06:02:24.656049+0000","last_undegraded":"2023-10-09T06:02:24.656049+0000","last_fullsized":"2023-10-09T06:02:24.656049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651456+0000","last_change":"2023-10-09T06:01:50.651743+0000","last_active":"2023-10-09T06:02:24.651456+0000","last_peered":"2023-10-09T06:02:24.651456+0000","last_clean":"2023-10-09T06:02:24.651456+0000","last_became_active":"2023-10-09T06:01:50.651429+0000","last_became_peered":"2023-10-09T06:01:50.651429+0000","last_unstale":"2023-10-09T06:02:24.651456+0000","last_undegraded":"2023-10-09T06:02:24.651456+0000","last_fullsized":"2023-10-09T06:02:24.651456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648601+0000","last_change":"2023-10-09T06:01:51.477427+0000","last_active":"2023-10-09T06:02:24.648601+0000","last_peered":"2023-10-09T06:02:24.648601+0000","last_clean":"2023-10-09T06:02:24.648601+0000","last_became_active":"2023-10-09T06:01:51.476469+0000","last_became_peered":"2023-10-09T06:01:51.476469+0000","last_unstale":"2023-10-09T06:02:24.648601+0000","last_undegraded":"2023-10-09T06:02:24.648601+0000","last_fullsized":"2023-10-09T06:02:24.648601+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648066+0000","last_change":"2023-10-09T06:01:50.653479+0000","last_active":"2023-10-09T06:02:24.648066+0000","last_peered":"2023-10-09T06:02:24.648066+0000","last_clean":"2023-10-09T06:02:24.648066+0000","last_became_active":"2023-10-09T06:01:50.653159+0000","last_became_peered":"2023-10-09T06:01:50.653159+0000","last_unstale":"2023-10-09T06:02:24.648066+0000","last_undegraded":"2023-10-09T06:02:24.648066+0000","last_fullsized":"2023-10-09T06:02:24.648066+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648793+0000","last_change":"2023-10-09T06:01:51.477429+0000","last_active":"2023-10-09T06:02:24.648793+0000","last_peered":"2023-10-09T06:02:24.648793+0000","last_clean":"2023-10-09T06:02:24.648793+0000","last_became_active":"2023-10-09T06:01:51.476565+0000","last_became_peered":"2023-10-09T06:01:51.476565+0000","last_unstale":"2023-10-09T06:02:24.648793+0000","last_undegraded":"2023-10-09T06:02:24.648793+0000","last_fullsized":"2023-10-09T06:02:24.648793+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654914+0000","last_change":"2023-10-09T06:01:50.654462+0000","last_active":"2023-10-09T06:02:24.654914+0000","last_peered":"2023-10-09T06:02:24.654914+0000","last_clean":"2023-10-09T06:02:24.654914+0000","last_became_active":"2023-10-09T06:01:50.654120+0000","last_became_peered":"2023-10-09T06:01:50.654120+0000","last_unstale":"2023-10-09T06:02:24.654914+0000","last_undegraded":"2023-10-09T06:02:24.654914+0000","last_fullsized":"2023-10-09T06:02:24.654914+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648174+0000","last_change":"2023-10-09T06:01:50.672468+0000","last_active":"2023-10-09T06:02:24.648174+0000","last_peered":"2023-10-09T06:02:24.648174+0000","last_clean":"2023-10-09T06:02:24.648174+0000","last_became_active":"2023-10-09T06:01:50.672182+0000","last_became_peered":"2023-10-09T06:01:50.672182+0000","last_unstale":"2023-10-09T06:02:24.648174+0000","last_undegraded":"2023-10-09T06:02:24.648174+0000","last_fullsized":"2023-10-09T06:02:24.648174+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651964+0000","last_change":"2023-10-09T06:01:50.655598+0000","last_active":"2023-10-09T06:02:24.651964+0000","last_peered":"2023-10-09T06:02:24.651964+0000","last_clean":"2023-10-09T06:02:24.651964+0000","last_became_active":"2023-10-09T06:01:50.655023+0000","last_became_peered":"2023-10-09T06:01:50.655023+0000","last_unstale":"2023-10-09T06:02:24.651964+0000","last_undegraded":"2023-10-09T06:02:24.651964+0000","last_fullsized":"2023-10-09T06:02:24.651964+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645525+0000","last_change":"2023-10-09T06:01:51.475825+0000","last_active":"2023-10-09T06:02:24.645525+0000","last_peered":"2023-10-09T06:02:24.645525+0000","last_clean":"2023-10-09T06:02:24.645525+0000","last_became_active":"2023-10-09T06:01:51.475392+0000","last_became_peered":"2023-10-09T06:01:51.475392+0000","last_unstale":"2023-10-09T06:02:24.645525+0000","last_undegraded":"2023-10-09T06:02:24.645525+0000","last_fullsized":"2023-10-09T06:02:24.645525+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656810+0000","last_change":"2023-10-09T06:01:50.649997+0000","last_active":"2023-10-09T06:02:24.656810+0000","last_peered":"2023-10-09T06:02:24.656810+0000","last_clean":"2023-10-09T06:02:24.656810+0000","last_became_active":"2023-10-09T06:01:50.649470+0000","last_became_peered":"2023-10-09T06:01:50.649470+0000","last_unstale":"2023-10-09T06:02:24.656810+0000","last_undegraded":"2023-10-09T06:02:24.656810+0000","last_fullsized":"2023-10-09T06:02:24.656810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.654761+0000","last_change":"2023-10-09T06:01:51.475354+0000","last_active":"2023-10-09T06:02:24.654761+0000","last_peered":"2023-10-09T06:02:24.654761+0000","last_clean":"2023-10-09T06:02:24.654761+0000","last_became_active":"2023-10-09T06:01:51.474419+0000","last_became_peered":"2023-10-09T06:01:51.474419+0000","last_unstale":"2023-10-09T06:02:24.654761+0000","last_undegraded":"2023-10-09T06:02:24.654761+0000","last_fullsized":"2023-10-09T06:02:24.654761+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649163+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:02:24.649163+0000","last_peered":"2023-10-09T06:02:24.649163+0000","last_clean":"2023-10-09T06:02:24.649163+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:02:24.649163+0000","last_undegraded":"2023-10-09T06:02:24.649163+0000","last_fullsized":"2023-10-09T06:02:24.649163+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648267+0000","last_change":"2023-10-09T06:01:51.478746+0000","last_active":"2023-10-09T06:02:24.648267+0000","last_peered":"2023-10-09T06:02:24.648267+0000","last_clean":"2023-10-09T06:02:24.648267+0000","last_became_active":"2023-10-09T06:01:51.477825+0000","last_became_peered":"2023-10-09T06:01:51.477825+0000","last_unstale":"2023-10-09T06:02:24.648267+0000","last_undegraded":"2023-10-09T06:02:24.648267+0000","last_fullsized":"2023-10-09T06:02:24.648267+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649132+0000","last_change":"2023-10-09T06:01:50.649059+0000","last_active":"2023-10-09T06:02:24.649132+0000","last_peered":"2023-10-09T06:02:24.649132+0000","last_clean":"2023-10-09T06:02:24.649132+0000","last_became_active":"2023-10-09T06:01:50.648710+0000","last_became_peered":"2023-10-09T06:01:50.648710+0000","last_unstale":"2023-10-09T06:02:24.649132+0000","last_undegraded":"2023-10-09T06:02:24.649132+0000","last_fullsized":"2023-10-09T06:02:24.649132+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656688+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:02:24.656688+0000","last_peered":"2023-10-09T06:02:24.656688+0000","last_clean":"2023-10-09T06:02:24.656688+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:02:24.656688+0000","last_undegraded":"2023-10-09T06:02:24.656688+0000","last_fullsized":"2023-10-09T06:02:24.656688+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649732+0000","last_change":"2023-10-09T06:01:50.651585+0000","last_active":"2023-10-09T06:02:24.649732+0000","last_peered":"2023-10-09T06:02:24.649732+0000","last_clean":"2023-10-09T06:02:24.649732+0000","last_became_active":"2023-10-09T06:01:50.651258+0000","last_became_peered":"2023-10-09T06:01:50.651258+0000","last_unstale":"2023-10-09T06:02:24.649732+0000","last_undegraded":"2023-10-09T06:02:24.649732+0000","last_fullsized":"2023-10-09T06:02:24.649732+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:25.658348+0000","last_change":"2023-10-09T06:02:25.658348+0000","last_active":"2023-10-09T06:02:25.658348+0000","last_peered":"2023-10-09T06:02:25.658348+0000","last_clean":"2023-10-09T06:02:25.658348+0000","last_became_active":"2023-10-09T06:02:25.657886+0000","last_became_peered":"2023-10-09T06:02:25.657886+0000","last_unstale":"2023-10-09T06:02:25.658348+0000","last_undegraded":"2023-10-09T06:02:25.658348+0000","last_fullsized":"2023-10-09T06:02:25.658348+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655225+0000","last_change":"2023-10-09T06:01:50.647073+0000","last_active":"2023-10-09T06:02:24.655225+0000","last_peered":"2023-10-09T06:02:24.655225+0000","last_clean":"2023-10-09T06:02:24.655225+0000","last_became_active":"2023-10-09T06:01:50.646739+0000","last_became_peered":"2023-10-09T06:01:50.646739+0000","last_unstale":"2023-10-09T06:02:24.655225+0000","last_undegraded":"2023-10-09T06:02:24.655225+0000","last_fullsized":"2023-10-09T06:02:24.655225+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645798+0000","last_change":"2023-10-09T06:01:50.672874+0000","last_active":"2023-10-09T06:02:24.645798+0000","last_peered":"2023-10-09T06:02:24.645798+0000","last_clean":"2023-10-09T06:02:24.645798+0000","last_became_active":"2023-10-09T06:01:50.672498+0000","last_became_peered":"2023-10-09T06:01:50.672498+0000","last_unstale":"2023-10-09T06:02:24.645798+0000","last_undegraded":"2023-10-09T06:02:24.645798+0000","last_fullsized":"2023-10-09T06:02:24.645798+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649238+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:02:24.649238+0000","last_peered":"2023-10-09T06:02:24.649238+0000","last_clean":"2023-10-09T06:02:24.649238+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:02:24.649238+0000","last_undegraded":"2023-10-09T06:02:24.649238+0000","last_fullsized":"2023-10-09T06:02:24.649238+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656968+0000","last_change":"2023-10-09T06:01:50.671509+0000","last_active":"2023-10-09T06:02:24.656968+0000","last_peered":"2023-10-09T06:02:24.656968+0000","last_clean":"2023-10-09T06:02:24.656968+0000","last_became_active":"2023-10-09T06:01:50.671114+0000","last_became_peered":"2023-10-09T06:01:50.671114+0000","last_unstale":"2023-10-09T06:02:24.656968+0000","last_undegraded":"2023-10-09T06:02:24.656968+0000","last_fullsized":"2023-10-09T06:02:24.656968+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645295+0000","last_change":"2023-10-09T06:01:50.658315+0000","last_active":"2023-10-09T06:02:24.645295+0000","last_peered":"2023-10-09T06:02:24.645295+0000","last_clean":"2023-10-09T06:02:24.645295+0000","last_became_active":"2023-10-09T06:01:50.657718+0000","last_became_peered":"2023-10-09T06:01:50.657718+0000","last_unstale":"2023-10-09T06:02:24.645295+0000","last_undegraded":"2023-10-09T06:02:24.645295+0000","last_fullsized":"2023-10-09T06:02:24.645295+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.652349+0000","last_change":"2023-10-09T06:01:51.477867+0000","last_active":"2023-10-09T06:02:24.652349+0000","last_peered":"2023-10-09T06:02:24.652349+0000","last_clean":"2023-10-09T06:02:24.652349+0000","last_became_active":"2023-10-09T06:01:51.477333+0000","last_became_peered":"2023-10-09T06:01:51.477333+0000","last_unstale":"2023-10-09T06:02:24.652349+0000","last_undegraded":"2023-10-09T06:02:24.652349+0000","last_fullsized":"2023-10-09T06:02:24.652349+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649411+0000","last_change":"2023-10-09T06:01:50.652517+0000","last_active":"2023-10-09T06:02:24.649411+0000","last_peered":"2023-10-09T06:02:24.649411+0000","last_clean":"2023-10-09T06:02:24.649411+0000","last_became_active":"2023-10-09T06:01:50.652208+0000","last_became_peered":"2023-10-09T06:01:50.652208+0000","last_unstale":"2023-10-09T06:02:24.649411+0000","last_undegraded":"2023-10-09T06:02:24.649411+0000","last_fullsized":"2023-10-09T06:02:24.649411+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.648098+0000","last_change":"2023-10-09T06:01:51.477776+0000","last_active":"2023-10-09T06:02:24.648098+0000","last_peered":"2023-10-09T06:02:24.648098+0000","last_clean":"2023-10-09T06:02:24.648098+0000","last_became_active":"2023-10-09T06:01:51.476653+0000","last_became_peered":"2023-10-09T06:01:51.476653+0000","last_unstale":"2023-10-09T06:02:24.648098+0000","last_undegraded":"2023-10-09T06:02:24.648098+0000","last_fullsized":"2023-10-09T06:02:24.648098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647900+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:02:24.647900+0000","last_peered":"2023-10-09T06:02:24.647900+0000","last_clean":"2023-10-09T06:02:24.647900+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:02:24.647900+0000","last_undegraded":"2023-10-09T06:02:24.647900+0000","last_fullsized":"2023-10-09T06:02:24.647900+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655373+0000","last_change":"2023-10-09T06:01:51.770691+0000","last_active":"2023-10-09T06:02:24.655373+0000","last_peered":"2023-10-09T06:02:24.655373+0000","last_clean":"2023-10-09T06:02:24.655373+0000","last_became_active":"2023-10-09T06:01:51.770218+0000","last_became_peered":"2023-10-09T06:01:51.770218+0000","last_unstale":"2023-10-09T06:02:24.655373+0000","last_undegraded":"2023-10-09T06:02:24.655373+0000","last_fullsized":"2023-10-09T06:02:24.655373+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.649055+0000","last_change":"2023-10-09T06:01:50.654185+0000","last_active":"2023-10-09T06:02:24.649055+0000","last_peered":"2023-10-09T06:02:24.649055+0000","last_clean":"2023-10-09T06:02:24.649055+0000","last_became_active":"2023-10-09T06:01:50.653699+0000","last_became_peered":"2023-10-09T06:01:50.653699+0000","last_unstale":"2023-10-09T06:02:24.649055+0000","last_undegraded":"2023-10-09T06:02:24.649055+0000","last_fullsized":"2023-10-09T06:02:24.649055+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651723+0000","last_change":"2023-10-09T06:01:50.678125+0000","last_active":"2023-10-09T06:02:24.651723+0000","last_peered":"2023-10-09T06:02:24.651723+0000","last_clean":"2023-10-09T06:02:24.651723+0000","last_became_active":"2023-10-09T06:01:50.677766+0000","last_became_peered":"2023-10-09T06:01:50.677766+0000","last_unstale":"2023-10-09T06:02:24.651723+0000","last_undegraded":"2023-10-09T06:02:24.651723+0000","last_fullsized":"2023-10-09T06:02:24.651723+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.656948+0000","last_change":"2023-10-09T06:01:50.651500+0000","last_active":"2023-10-09T06:02:24.656948+0000","last_peered":"2023-10-09T06:02:24.656948+0000","last_clean":"2023-10-09T06:02:24.656948+0000","last_became_active":"2023-10-09T06:01:50.651172+0000","last_became_peered":"2023-10-09T06:01:50.651172+0000","last_unstale":"2023-10-09T06:02:24.656948+0000","last_undegraded":"2023-10-09T06:02:24.656948+0000","last_fullsized":"2023-10-09T06:02:24.656948+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T06:02:26.083506+0000","last_change":"2023-10-09T06:02:26.083506+0000","last_active":"2023-10-09T06:02:26.083506+0000","last_peered":"2023-10-09T06:02:26.083506+0000","last_clean":"2023-10-09T06:02:26.083506+0000","last_became_active":"2023-10-09T06:02:26.082579+0000","last_became_peered":"2023-10-09T06:02:26.082579+0000","last_unstale":"2023-10-09T06:02:26.083506+0000","last_undegraded":"2023-10-09T06:02:26.083506+0000","last_fullsized":"2023-10-09T06:02:26.083506+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.646302+0000","last_change":"2023-10-09T06:01:50.658448+0000","last_active":"2023-10-09T06:02:24.646302+0000","last_peered":"2023-10-09T06:02:24.646302+0000","last_clean":"2023-10-09T06:02:24.646302+0000","last_became_active":"2023-10-09T06:01:50.658007+0000","last_became_peered":"2023-10-09T06:01:50.658007+0000","last_unstale":"2023-10-09T06:02:24.646302+0000","last_undegraded":"2023-10-09T06:02:24.646302+0000","last_fullsized":"2023-10-09T06:02:24.646302+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655157+0000","last_change":"2023-10-09T06:01:51.477390+0000","last_active":"2023-10-09T06:02:24.655157+0000","last_peered":"2023-10-09T06:02:24.655157+0000","last_clean":"2023-10-09T06:02:24.655157+0000","last_became_active":"2023-10-09T06:01:51.476482+0000","last_became_peered":"2023-10-09T06:01:51.476482+0000","last_unstale":"2023-10-09T06:02:24.655157+0000","last_undegraded":"2023-10-09T06:02:24.655157+0000","last_fullsized":"2023-10-09T06:02:24.655157+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651615+0000","last_change":"2023-10-09T06:01:50.655973+0000","last_active":"2023-10-09T06:02:24.651615+0000","last_peered":"2023-10-09T06:02:24.651615+0000","last_clean":"2023-10-09T06:02:24.651615+0000","last_became_active":"2023-10-09T06:01:50.655664+0000","last_became_peered":"2023-10-09T06:01:50.655664+0000","last_unstale":"2023-10-09T06:02:24.651615+0000","last_undegraded":"2023-10-09T06:02:24.651615+0000","last_fullsized":"2023-10-09T06:02:24.651615+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.645797+0000","last_change":"2023-10-09T06:01:50.674862+0000","last_active":"2023-10-09T06:02:24.645797+0000","last_peered":"2023-10-09T06:02:24.645797+0000","last_clean":"2023-10-09T06:02:24.645797+0000","last_became_active":"2023-10-09T06:01:50.674566+0000","last_became_peered":"2023-10-09T06:01:50.674566+0000","last_unstale":"2023-10-09T06:02:24.645797+0000","last_undegraded":"2023-10-09T06:02:24.645797+0000","last_fullsized":"2023-10-09T06:02:24.645797+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.655375+0000","last_change":"2023-10-09T06:01:50.655797+0000","last_active":"2023-10-09T06:02:24.655375+0000","last_peered":"2023-10-09T06:02:24.655375+0000","last_clean":"2023-10-09T06:02:24.655375+0000","last_became_active":"2023-10-09T06:01:50.655032+0000","last_became_peered":"2023-10-09T06:01:50.655032+0000","last_unstale":"2023-10-09T06:02:24.655375+0000","last_undegraded":"2023-10-09T06:02:24.655375+0000","last_fullsized":"2023-10-09T06:02:24.655375+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.651985+0000","last_change":"2023-10-09T06:01:51.473771+0000","last_active":"2023-10-09T06:02:24.651985+0000","last_peered":"2023-10-09T06:02:24.651985+0000","last_clean":"2023-10-09T06:02:24.651985+0000","last_became_active":"2023-10-09T06:01:51.473476+0000","last_became_peered":"2023-10-09T06:01:51.473476+0000","last_unstale":"2023-10-09T06:02:24.651985+0000","last_undegraded":"2023-10-09T06:02:24.651985+0000","last_fullsized":"2023-10-09T06:02:24.651985+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-09T06:02:24.647784+0000","last_change":"2023-10-09T06:01:50.657344+0000","last_active":"2023-10-09T06:02:24.647784+0000","last_peered":"2023-10-09T06:02:24.647784+0000","last_clean":"2023-10-09T06:02:24.647784+0000","last_became_active":"2023-10-09T06:01:50.657015+0000","last_became_peered":"2023-10-09T06:01:50.657015+0000","last_unstale":"2023-10-09T06:02:24.647784+0000","last_undegraded":"2023-10-09T06:02:24.647784+0000","last_fullsized":"2023-10-09T06:02:24.647784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430267,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.85799999999999998,"15min":0.878},"min":{"1min":0.60699999999999998,"5min":0.371,"15min":0.371},"max":{"1min":1.274,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.93000000000000005,"15min":0.93400000000000005},"min":{"1min":0.70399999999999996,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2549999999999999,"5min":1.452,"15min":1.452},"last":0.83999999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.88700000000000001,"15min":0.89200000000000002},"min":{"1min":0.64800000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"max":{"1min":1.3080000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.94399999999999995,"15min":0.97399999999999998},"min":{"1min":0.69399999999999995,"5min":0.67800000000000005,"15min":0.67800000000000005},"max":{"1min":1.216,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":1.01}]},{"osd":2,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.88700000000000001,"15min":0.90000000000000002},"min":{"1min":0.68000000000000005,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.90700000000000003,"15min":0.90300000000000002},"min":{"1min":0.69499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1779999999999999,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.085}]},{"osd":3,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.93899999999999995,"15min":0.95999999999999996},"min":{"1min":0.65600000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.24,"5min":1.335,"15min":1.335},"last":1.2609999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.94799999999999995,"15min":0.95399999999999996},"min":{"1min":0.75800000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.248,"5min":1.5609999999999999,"15min":1.5609999999999999},"last":1.1859999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.93999999999999995,"15min":0.95499999999999996},"min":{"1min":0.65600000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3149999999999999,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.129},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.90500000000000003,"15min":0.90300000000000002},"min":{"1min":0.53600000000000003,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.2,"5min":1.51,"15min":1.51},"last":1.0509999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.96699999999999997,"15min":0.98099999999999998},"min":{"1min":0.60099999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.359,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.03,"15min":1.022},"min":{"1min":0.874,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3939999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.2949999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.0660000000000001,"15min":1.0569999999999999},"min":{"1min":0.88900000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.4399999999999999,"5min":1.508,"15min":1.508},"last":1.347},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.044,"15min":1.044},"min":{"1min":0.85099999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.2989999999999999,"5min":2.0019999999999998,"15min":2.0019999999999998},"last":1.1579999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626499,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.873,"15min":0.88300000000000001},"min":{"1min":0.51800000000000002,"5min":0.439,"15min":0.439},"max":{"1min":1.1299999999999999,"5min":1.294,"15min":1.294},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.873,"15min":0.88900000000000001},"min":{"1min":0.54800000000000004,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3160000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0529999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83799999999999997,"15min":0.83299999999999996},"min":{"1min":0.52700000000000002,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.3680000000000001,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.86899999999999999,"15min":0.83099999999999996},"min":{"1min":0.59199999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.137,"5min":1.278,"15min":1.278},"last":0.90200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.90300000000000002,"15min":0.878},"min":{"1min":0.67300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.2150000000000001,"5min":1.423,"15min":1.423},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.93799999999999994,"15min":0.96499999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.367,"5min":1.786,"15min":1.786},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.88500000000000001,"15min":0.86799999999999999},"min":{"1min":0.63,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.03,"5min":0.99299999999999999,"15min":0.95699999999999996},"min":{"1min":0.68999999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3400000000000001,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":1.085}]},{"osd":4,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.93000000000000005,"15min":0.89300000000000002},"min":{"1min":0.77100000000000002,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.504,"5min":1.6879999999999999,"15min":1.6879999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.083,"5min":0.95199999999999996,"15min":0.90100000000000002},"min":{"1min":0.81299999999999994,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.3149999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.68000000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.97999999999999998,"15min":0.878},"min":{"1min":0.88400000000000001,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2749999999999999,"5min":1.708,"15min":1.708},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":0.997,"15min":0.93200000000000005},"min":{"1min":0.70599999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.4099999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.93999999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:03:00 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.075,"15min":1.0349999999999999},"min":{"1min":0.87,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4410000000000001,"5min":1.629,"15min":1.629},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.087,"15min":1.052},"min":{"1min":0.73099999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.4550000000000001,"5min":1.47,"15min":1.47},"last":1.0209999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574960,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":269625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.70999999999999996,"15min":0.628},"min":{"1min":0.375,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":1.1519999999999999,"5min":1.3600000000000001,"15min":1.5369999999999999},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.751,"5min":0.69699999999999995,"15min":0.626},"min":{"1min":0.51400000000000001,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":0.97799999999999998,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.68899999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.75,"15min":0.67300000000000004},"min":{"1min":0.47199999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.96899999999999997,"5min":1.464,"15min":1.464},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.745,"5min":0.72699999999999998,"15min":0.66800000000000004},"min":{"1min":0.49399999999999999,"5min":0.37,"15min":0.37},"max":{"1min":1.012,"5min":1.234,"15min":1.234},"last":0.83499999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.78100000000000003,"15min":0.80100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.423,"5min":1.423,"15min":1.5820000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.746,"15min":0.74199999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.125,"5min":1.125,"15min":1.3480000000000001},"last":0.76800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:02:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.89000000000000001,"15min":0.89100000000000001},"min":{"1min":0.62,"5min":0.56799999999999995,"15min":0.54700000000000004},"max":{"1min":1.0940000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.042},{"interface":"front","average":{"1min":1.03,"5min":1.0109999999999999,"15min":0.99099999999999999},"min":{"1min":0.68200000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.3300000000000001,"5min":1.575,"15min":1.6419999999999999},"last":0.745}]},{"osd":5,"last update":"Mon Oct 9 06:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.96599999999999997,"15min":0.95599999999999996},"min":{"1min":0.753,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.2829999999999999,"5min":1.294,"15min":1.294},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.97999999999999998,"15min":0.96499999999999997},"min":{"1min":0.81999999999999995,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2909999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.107}]},{"osd":6,"last update":"Mon Oct 9 06:03:21 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.0780000000000001,"15min":1.0900000000000001},"min":{"1min":0.624,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.4890000000000001,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":1.008},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.016,"15min":1.044},"min":{"1min":0.60999999999999999,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.3560000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.0700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0840000000000001,"15min":1.1120000000000001},"min":{"1min":0.70999999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.4430000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1739999999999999,"15min":1.212},"min":{"1min":0.71899999999999997,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.54,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":1.0569999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738488,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.63,"15min":0.57099999999999995},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.34699999999999998},"max":{"1min":0.96499999999999997,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.73199999999999998,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.167,"5min":1.1930000000000001,"15min":1.22},"last":0.57199999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.69699999999999995,"15min":0.67400000000000004},"min":{"1min":0.35999999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0109999999999999,"5min":1.2090000000000001,"15min":1.3089999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.67500000000000004,"15min":0.64600000000000002},"min":{"1min":0.33600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.008,"5min":1.413,"15min":1.413},"last":0.79200000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:03:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.76800000000000002,"15min":0.79400000000000004},"min":{"1min":0.41999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.1259999999999999,"5min":1.1519999999999999,"15min":1.387},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.754,"15min":0.78100000000000003},"min":{"1min":0.39200000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.85499999999999998,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.876}]},{"osd":4,"last update":"Mon Oct 9 06:02:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.88,"15min":0.84999999999999998},"min":{"1min":0.67900000000000005,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.206,"5min":1.7889999999999999,"15min":1.7889999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.498,"5min":1.0800000000000001,"15min":1.0209999999999999},"min":{"1min":0.75700000000000001,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":9.0280000000000005,"5min":9.0280000000000005,"15min":9.0280000000000005},"last":1.194}]},{"osd":5,"last update":"Mon Oct 9 06:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.995,"15min":0.99099999999999999},"min":{"1min":0.59999999999999998,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4139999999999999,"5min":1.968,"15min":1.968},"last":1.008},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.006,"15min":1.016},"min":{"1min":0.66300000000000003,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.403,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.93000000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:03:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.018,"15min":1.1040000000000001},"min":{"1min":0.65400000000000003,"5min":0.505,"15min":0.505},"max":{"1min":1.401,"5min":2.0409999999999999,"15min":2.0409999999999999},"last":1.133},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.075,"15min":1.075},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":1.387,"5min":6.7290000000000001,"15min":6.7290000000000001},"last":1.2250000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:55 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.0189999999999999,"15min":1.044},"min":{"1min":0.627,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.472,"5min":1.9299999999999999,"15min":1.9299999999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.335,"5min":1.1200000000000001,"15min":1.0880000000000001},"min":{"1min":0.74099999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":5.1820000000000004,"5min":5.1820000000000004,"15min":5.1820000000000004},"last":1.2929999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411431,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.67500000000000004,"15min":0.63300000000000001},"min":{"1min":0.31900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":1.171,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.63200000000000001},"min":{"1min":0.41899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.95099999999999996,"5min":2.2719999999999998,"15min":2.2719999999999998},"last":0.64600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.70699999999999996,"15min":0.64600000000000002},"min":{"1min":0.45000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.03,"5min":1.496,"15min":1.496},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.746,"15min":0.69599999999999995},"min":{"1min":0.51000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":1.022,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.74299999999999999,"15min":0.76100000000000001},"min":{"1min":0.42199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0609999999999999,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.77100000000000002,"15min":0.78400000000000003},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.123,"5min":1.546,"15min":1.546},"last":0.68200000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:02:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.95599999999999996,"15min":0.93100000000000005},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.9810000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":1.087},{"interface":"front","average":{"1min":1.095,"5min":1.006,"15min":1.032},"min":{"1min":0.58799999999999997,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.97499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:02:31 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.95899999999999996,"15min":0.98199999999999998},"min":{"1min":0.64400000000000002,"5min":0.505,"15min":0.505},"max":{"1min":1.4199999999999999,"5min":1.734,"15min":1.734},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.022,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":2.2669999999999999,"5min":2.2669999999999999,"15min":2.2669999999999999},"last":0.78900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:03:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.012,"15min":1.042},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.3300000000000001,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":1.018},{"interface":"front","average":{"1min":1.111,"5min":1.083,"15min":1.0920000000000001},"min":{"1min":0.70699999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.92800000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.054,"15min":1.04},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.8740000000000001,"5min":2.1160000000000001,"15min":2.1160000000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.0680000000000001,"15min":1.056},"min":{"1min":0.628,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":2.21,"5min":2.21,"15min":2.21},"last":0.76400000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182493,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.33300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.0740000000000001,"5min":1.232,"15min":1.232},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.69899999999999995,"15min":0.72399999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81799999999999995,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.71899999999999997,"15min":0.77600000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.1910000000000001,"5min":1.7749999999999999,"15min":1.7749999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.68999999999999995,"15min":0.76300000000000001},"min":{"1min":0.40000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.036,"5min":1.036,"15min":1.4319999999999999},"last":0.66900000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.73499999999999999,"15min":0.75800000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.0029999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.76800000000000002,"15min":0.79300000000000004},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.429,"5min":1.754,"15min":1.754},"last":0.91100000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:02:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.93899999999999995,"15min":0.90600000000000003},"min":{"1min":0.68600000000000005,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3380000000000001,"5min":1.6699999999999999,"15min":1.6699999999999999},"last":1.161},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97499999999999998,"15min":0.93100000000000005},"min":{"1min":0.76800000000000002,"5min":0.622,"15min":0.622},"max":{"1min":1.2010000000000001,"5min":1.496,"15min":1.496},"last":0.93700000000000006}]},{"osd":5,"last update":"Mon Oct 9 06:02:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.006,"15min":1.036},"min":{"1min":0.72199999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.325,"5min":1.556,"15min":1.556},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.016,"15min":1.0289999999999999},"min":{"1min":0.84399999999999997,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.2529999999999999,"5min":1.651,"15min":1.651},"last":1.1120000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:03:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0109999999999999,"15min":1.093},"min":{"1min":0.69899999999999995,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.24},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.046,"15min":1.1100000000000001},"min":{"1min":0.63300000000000001,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.2509999999999999,"5min":1.609,"15min":1.609},"last":1.0780000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:02:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0860000000000001,"15min":1.1200000000000001},"min":{"1min":0.75,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.534,"5min":1.534,"15min":1.534},"last":1.323},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.111,"15min":1.1379999999999999},"min":{"1min":0.747,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.361,"5min":1.665,"15min":1.665},"last":1.2549999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018965,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86699999999999999,"15min":0.84699999999999998},"min":{"1min":0.57399999999999995,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.2809999999999999,"5min":1.8049999999999999,"15min":1.8049999999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.016,"5min":0.91000000000000003,"15min":0.84399999999999997},"min":{"1min":0.753,"5min":0.52000000000000002,"15min":0.41999999999999998},"max":{"1min":1.419,"5min":1.927,"15min":1.927},"last":0.83099999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.88300000000000001,"15min":0.81499999999999995},"min":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3200000000000001,"5min":2.3410000000000002,"15min":2.3410000000000002},"last":0.997},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.95499999999999996,"15min":0.88700000000000001},"min":{"1min":0.58299999999999996,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.524,"5min":3.3690000000000002,"15min":3.3690000000000002},"last":1.157}]},{"osd":2,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.94799999999999995,"15min":0.89600000000000002},"min":{"1min":0.61099999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.93400000000000005,"15min":0.88800000000000001},"min":{"1min":0.64100000000000001,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.349,"5min":2.1150000000000002,"15min":2.1150000000000002},"last":1.1870000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.96399999999999997,"15min":0.88800000000000001},"min":{"1min":0.54900000000000004,"5min":0.497,"15min":0.497},"max":{"1min":1.4870000000000001,"5min":3.0760000000000001,"15min":3.0760000000000001},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.98299999999999998,"15min":0.89300000000000002},"min":{"1min":0.68799999999999994,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.238,"5min":2.5619999999999998,"15min":2.5619999999999998},"last":1.1259999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:02:30 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.877,"15min":0.77700000000000002},"min":{"1min":0.61799999999999999,"5min":0.372,"15min":0.372},"max":{"1min":1.869,"5min":2.79,"15min":2.79},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.90900000000000003,"15min":0.79300000000000004},"min":{"1min":0.75800000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":1.05}]},{"osd":6,"last update":"Mon Oct 9 06:03:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.998,"15min":0.97099999999999997},"min":{"1min":0.71299999999999997,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3009999999999999,"5min":1.891,"15min":1.891},"last":1.264},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.022,"15min":0.99099999999999999},"min":{"1min":0.80500000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.248,"5min":2.915,"15min":2.915},"last":1.2350000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:03:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.085,"15min":1.077},"min":{"1min":0.77800000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3740000000000001,"5min":2.056,"15min":2.056},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.0920000000000001,"15min":1.081},"min":{"1min":0.83399999999999996,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.3560000000000001,"5min":2.629,"15min":2.629},"last":1.286}]}]},{"osd":5,"up_from":36,"seq":154618822732,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.872,"15min":0.86499999999999999},"min":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3420000000000001,"5min":1.492,"15min":1.492},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.89000000000000001,"15min":0.874},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.85599999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.90700000000000003,"15min":0.88900000000000001},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3049999999999999,"5min":1.377,"15min":1.377},"last":0.749},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.88600000000000001,"15min":0.89400000000000002},"min":{"1min":0.57199999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.1779999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.1319999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.89000000000000001,"15min":0.89400000000000002},"min":{"1min":0.66600000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.93999999999999995},"min":{"1min":0.72999999999999998,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3859999999999999,"5min":1.609,"15min":1.609},"last":0.93700000000000006}]},{"osd":3,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.91100000000000003,"15min":0.88700000000000001},"min":{"1min":0.71799999999999997,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.22,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.012,"5min":0.98799999999999999,"15min":0.97399999999999998},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.349,"5min":1.7370000000000001,"15min":1.7370000000000001},"last":1.024}]},{"osd":4,"last update":"Mon Oct 9 06:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.87,"15min":0.82199999999999995},"min":{"1min":0.65000000000000002,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.3080000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.89400000000000002,"15min":0.80800000000000005},"min":{"1min":0.63200000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.536,"5min":1.718,"15min":1.718},"last":1.228}]},{"osd":6,"last update":"Mon Oct 9 06:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.97399999999999998,"15min":0.96199999999999997},"min":{"1min":0.81200000000000006,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.365,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":1.208},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0149999999999999,"15min":1.0289999999999999},"min":{"1min":0.63800000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.385,"5min":1.6399999999999999,"15min":1.6399999999999999},"last":1.274}]},{"osd":7,"last update":"Mon Oct 9 06:02:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.0649999999999999,"15min":1.079},"min":{"1min":0.56200000000000006,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3899999999999999,"5min":1.718,"15min":1.718},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.101,"15min":1.1160000000000001},"min":{"1min":0.72299999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.3939999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.0760000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:03:34.123 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:03:34.123 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T06:03:34.123 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T06:03:34.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T06:03:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:34 smithi125 bash[27513]: audit 2023-10-09T06:03:33.334860+0000 mgr.y (mgr.14430) 159 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:34 smithi125 bash[33919]: audit 2023-10-09T06:03:33.334860+0000 mgr.y (mgr.14430) 159 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:34 smithi155 bash[27817]: audit 2023-10-09T06:03:33.334860+0000 mgr.y (mgr.14430) 159 : audit [DBG] from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:03:35.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:35 smithi125 bash[27513]: cluster 2023-10-09T06:03:34.255770+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:35 smithi125 bash[33919]: cluster 2023-10-09T06:03:34.255770+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:35 smithi155 bash[27817]: cluster 2023-10-09T06:03:34.255770+0000 mgr.y (mgr.14430) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:37.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:37 smithi125 bash[27513]: cluster 2023-10-09T06:03:36.256807+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:37 smithi125 bash[33919]: cluster 2023-10-09T06:03:36.256807+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:37 smithi155 bash[27817]: cluster 2023-10-09T06:03:36.256807+0000 mgr.y (mgr.14430) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:38.937 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:39.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:38 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:38] "GET /metrics HTTP/1.1" 200 191543 "" "Prometheus/2.33.4" 2023-10-09T06:03:39.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:39 smithi125 bash[27513]: cluster 2023-10-09T06:03:38.257411+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:39.847 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:39 smithi125 bash[33919]: cluster 2023-10-09T06:03:38.257411+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:39 smithi155 bash[27817]: cluster 2023-10-09T06:03:38.257411+0000 mgr.y (mgr.14430) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:41.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:41 smithi155 bash[27817]: cluster 2023-10-09T06:03:40.257984+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:41 smithi125 bash[27513]: cluster 2023-10-09T06:03:40.257984+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:41.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:41 smithi125 bash[33919]: cluster 2023-10-09T06:03:40.257984+0000 mgr.y (mgr.14430) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:41.974 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:03:42.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:41 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:42.791 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T06:03:42.791 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T06:03:42.792 INFO:teuthology.run_tasks:Running task print... 2023-10-09T06:03:42.801 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T06:03:42.801 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T06:03:42.823 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T06:03:42.824 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T06:03:42.824 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:03:42.828 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:03:42.828 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:03:42.829 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:03:42.829 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T06:03:42.829 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T06:03:42.829 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T06:03:42.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T06:03:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:03:42.837 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:03:42.837 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:03:42.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:42 smithi125 bash[27513]: audit 2023-10-09T06:03:41.972532+0000 mon.a (mon.0) 687 : audit [DBG] from='client.? 172.21.15.125:0/1828527720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:03:42.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:42 smithi125 bash[33919]: audit 2023-10-09T06:03:41.972532+0000 mon.a (mon.0) 687 : audit [DBG] from='client.? 172.21.15.125:0/1828527720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:03:42.890 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:03:42.890 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:03:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:42 smithi155 bash[27817]: audit 2023-10-09T06:03:41.972532+0000 mon.a (mon.0) 687 : audit [DBG] from='client.? 172.21.15.125:0/1828527720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:03:42.942 INFO:tasks.workunit:timeout=3h 2023-10-09T06:03:42.942 INFO:tasks.workunit:cleanup=True 2023-10-09T06:03:42.942 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:03:42.995 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:03:43.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:43 smithi125 bash[27513]: cluster 2023-10-09T06:03:42.258459+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:43 smithi125 bash[33919]: cluster 2023-10-09T06:03:42.258459+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:43 smithi155 bash[27817]: cluster 2023-10-09T06:03:42.258459+0000 mgr.y (mgr.14430) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:45.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:45 smithi125 bash[27513]: cluster 2023-10-09T06:03:44.258837+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:45.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:45 smithi125 bash[33919]: cluster 2023-10-09T06:03:44.258837+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:45 smithi155 bash[27817]: cluster 2023-10-09T06:03:44.258837+0000 mgr.y (mgr.14430) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:46.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE Bus STOPPING 2023-10-09T06:03:46.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:03:46.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE Bus STOPPED 2023-10-09T06:03:46.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE Bus STARTING 2023-10-09T06:03:46.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE Serving on http://:::9283 2023-10-09T06:03:46.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:46 smithi125 bash[27779]: [09/Oct/2023:06:03:46] ENGINE Bus STARTED 2023-10-09T06:03:47.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T06:03:47.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[33919]: audit 2023-10-09T06:03:46.170567+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.125:0/702559327' entity='client.admin' 2023-10-09T06:03:47.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[33919]: audit 2023-10-09T06:03:46.195395+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:47.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[33919]: cluster 2023-10-09T06:03:46.259519+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:47.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[27513]: audit 2023-10-09T06:03:46.170567+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.125:0/702559327' entity='client.admin' 2023-10-09T06:03:47.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[27513]: audit 2023-10-09T06:03:46.195395+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:47.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:47 smithi125 bash[27513]: cluster 2023-10-09T06:03:46.259519+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:47 smithi155 bash[27817]: audit 2023-10-09T06:03:46.170567+0000 mon.a (mon.0) 688 : audit [INF] from='client.? 172.21.15.125:0/702559327' entity='client.admin' 2023-10-09T06:03:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:47 smithi155 bash[27817]: audit 2023-10-09T06:03:46.195395+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:47 smithi155 bash[27817]: cluster 2023-10-09T06:03:46.259519+0000 mgr.y (mgr.14430) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:48.769 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:03:48.781 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:03:48.783 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:03:48.783 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:03:49.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:48] "GET /metrics HTTP/1.1" 200 191542 "" "Prometheus/2.33.4" 2023-10-09T06:03:49.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:49 smithi125 bash[27513]: cluster 2023-10-09T06:03:48.259884+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:49 smithi125 bash[33919]: cluster 2023-10-09T06:03:48.259884+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:49 smithi155 bash[27817]: cluster 2023-10-09T06:03:48.259884+0000 mgr.y (mgr.14430) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:51.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:51 smithi155 bash[27817]: cluster 2023-10-09T06:03:50.260825+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:51.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:51 smithi125 bash[27513]: cluster 2023-10-09T06:03:50.260825+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:51.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:51 smithi125 bash[33919]: cluster 2023-10-09T06:03:50.260825+0000 mgr.y (mgr.14430) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:03:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:03:52.981 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:03:53.063 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:03:53.117 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:03:53.119 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:03:53.119 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:03:53.176 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:03:53.180 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T06:03:53.181 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:03:53.234 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T06:03:53.235 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T06:03:53.236 DEBUG:teuthology.orchestra.run.smithi125:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T06:03:53.286 INFO:tasks.workunit.client.0.smithi125.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T06:03:53.287 INFO:tasks.workunit.client.0.smithi125.stderr:+ TMPDIR=/tmp/rbd_import_export_66110 2023-10-09T06:03:53.287 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_66110 2023-10-09T06:03:53.288 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir /tmp/rbd_import_export_66110 2023-10-09T06:03:53.289 INFO:tasks.workunit.client.0.smithi125.stderr:+ trap rm -rf /tmp/rbd_import_export_66110 INT TERM EXIT 2023-10-09T06:03:53.289 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir foo.66110 2023-10-09T06:03:53.291 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import foo.66110 foo.dir 2023-10-09T06:03:53.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:52.136166+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: cluster 2023-10-09T06:03:52.261277+0000 mgr.y (mgr.14430) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:53.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:52.914781+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:52.917066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:52.918089+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:52.923909+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:53.071559+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T06:03:53.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:53.095195+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:53.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:53.097825+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:53.098977+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27513]: audit 2023-10-09T06:03:53.107078+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:52.136166+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: cluster 2023-10-09T06:03:52.261277+0000 mgr.y (mgr.14430) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:52.914781+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:52.917066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:52.918089+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:52.923909+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:53.071559+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T06:03:53.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:53.095195+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:53.341 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:53.097825+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.341 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:53.098977+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.341 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[33919]: audit 2023-10-09T06:03:53.107078+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.341 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE Bus STOPPING 2023-10-09T06:03:53.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:03:53.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE Bus STOPPED 2023-10-09T06:03:53.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE Bus STARTING 2023-10-09T06:03:53.342 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE Serving on http://:::9283 2023-10-09T06:03:53.343 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:53 smithi125 bash[27779]: [09/Oct/2023:06:03:53] ENGINE Bus STARTED 2023-10-09T06:03:53.372 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: cannot import a directory 2023-10-09T06:03:53.372 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 0% complete...failed. 2023-10-09T06:03:53.372 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: import failed: (21) Is a directory 2023-10-09T06:03:53.376 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:03:53.376 INFO:tasks.workunit.client.0.smithi125.stderr:+ rmdir foo.66110 2023-10-09T06:03:53.607 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_66110/img bs=1k count=1 seek=10 2023-10-09T06:03:53.608 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:03:53.608 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:03:53.608 INFO:tasks.workunit.client.0.smithi125.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000195944 s, 5.2 MB/s 2023-10-09T06:03:53.608 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_66110/img bs=1k count=10 seek=100 2023-10-09T06:03:53.609 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T06:03:53.609 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T06:03:53.609 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000104365 s, 98.1 MB/s 2023-10-09T06:03:53.609 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_66110/img bs=1k count=100 seek=1000 2023-10-09T06:03:53.611 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records in 2023-10-09T06:03:53.611 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records out 2023-10-09T06:03:53.611 INFO:tasks.workunit.client.0.smithi125.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000593542 s, 121 MB/s 2023-10-09T06:03:53.611 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_66110/img bs=1k seek=10000 2023-10-09T06:03:53.613 INFO:tasks.workunit.client.0.smithi125.stderr:138+1 records in 2023-10-09T06:03:53.613 INFO:tasks.workunit.client.0.smithi125.stderr:138+1 records out 2023-10-09T06:03:53.614 INFO:tasks.workunit.client.0.smithi125.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.0011548 s, 123 MB/s 2023-10-09T06:03:53.614 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_66110/img bs=1k seek=100000 2023-10-09T06:03:53.614 INFO:tasks.workunit.client.0.smithi125.stderr:74+1 records in 2023-10-09T06:03:53.614 INFO:tasks.workunit.client.0.smithi125.stderr:74+1 records out 2023-10-09T06:03:53.615 INFO:tasks.workunit.client.0.smithi125.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.000263934 s, 289 MB/s 2023-10-09T06:03:53.615 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_66110/img bs=1k seek=1000000 2023-10-09T06:03:53.617 INFO:tasks.workunit.client.0.smithi125.stderr:194+1 records in 2023-10-09T06:03:53.617 INFO:tasks.workunit.client.0.smithi125.stderr:194+1 records out 2023-10-09T06:03:53.617 INFO:tasks.workunit.client.0.smithi125.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.000727287 s, 274 MB/s 2023-10-09T06:03:53.618 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:03:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:52.136166+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: cluster 2023-10-09T06:03:52.261277+0000 mgr.y (mgr.14430) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:52.914781+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:52.917066+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:52.918089+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:52.923909+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:53.071559+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:53.095195+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:53.097825+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:53.098977+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:53.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:53 smithi155 bash[27817]: audit 2023-10-09T06:03:53.107078+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:03:53.668 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:03:53.669 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:03:53.673 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:03:53.674 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:03:53.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T06:03:55.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:55 smithi125 bash[33919]: cluster 2023-10-09T06:03:54.149740+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T06:03:55.385 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:55 smithi125 bash[33919]: cluster 2023-10-09T06:03:54.261784+0000 mgr.y (mgr.14430) 170 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:55.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:55 smithi125 bash[27513]: cluster 2023-10-09T06:03:54.149740+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T06:03:55.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:55 smithi125 bash[27513]: cluster 2023-10-09T06:03:54.261784+0000 mgr.y (mgr.14430) 170 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:55 smithi155 bash[27817]: cluster 2023-10-09T06:03:54.149740+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-09T06:03:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:55 smithi155 bash[27817]: cluster 2023-10-09T06:03:54.261784+0000 mgr.y (mgr.14430) 170 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:55.784 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T06:03:55.785 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:03:55.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_66110/img2 2023-10-09T06:03:56.141 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:03:56.145 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T06:03:56.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:56 smithi125 bash[27513]: cluster 2023-10-09T06:03:55.159560+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T06:03:56.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:56 smithi125 bash[33919]: cluster 2023-10-09T06:03:55.159560+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T06:03:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:56 smithi155 bash[27817]: cluster 2023-10-09T06:03:55.159560+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-09T06:03:57.032 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:03:57.036 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:03:57.178 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:03:57.181 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img2 2023-10-09T06:03:57.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:57 smithi125 bash[27513]: cluster 2023-10-09T06:03:56.262840+0000 mgr.y (mgr.14430) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:03:57.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:57 smithi125 bash[33919]: cluster 2023-10-09T06:03:56.262840+0000 mgr.y (mgr.14430) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:03:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:57 smithi155 bash[27817]: cluster 2023-10-09T06:03:56.262840+0000 mgr.y (mgr.14430) 171 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:03:57.895 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img3 2023-10-09T06:03:58.412 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img2 /tmp/rbd_import_export_66110/img3 2023-10-09T06:03:59.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:03:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:03:58] "GET /metrics HTTP/1.1" 200 191545 "" "Prometheus/2.33.4" 2023-10-09T06:03:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:03:59 smithi125 bash[33919]: cluster 2023-10-09T06:03:58.263561+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:03:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:03:59 smithi125 bash[27513]: cluster 2023-10-09T06:03:58.263561+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:03:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:03:59 smithi155 bash[27817]: cluster 2023-10-09T06:03:58.263561+0000 mgr.y (mgr.14430) 172 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 3.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 30 KiB/s wr, 8 op/s 2023-10-09T06:04:01.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:01 smithi155 bash[27817]: cluster 2023-10-09T06:04:00.264581+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:01.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:01 smithi155 bash[27817]: cluster 2023-10-09T06:04:00.402784+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T06:04:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:01 smithi125 bash[27513]: cluster 2023-10-09T06:04:00.264581+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:01 smithi125 bash[27513]: cluster 2023-10-09T06:04:00.402784+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T06:04:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:01 smithi125 bash[33919]: cluster 2023-10-09T06:04:00.264581+0000 mgr.y (mgr.14430) 173 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:01 smithi125 bash[33919]: cluster 2023-10-09T06:04:00.402784+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-09T06:04:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:01 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:02.229 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format - testimg 2023-10-09T06:04:02.805 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:02.806 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:02.811 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_66110/img2 2023-10-09T06:04:03.139 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:03.144 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T06:04:03.264 INFO:teuthology.orchestra.run.smithi125.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:03.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: cluster 2023-10-09T06:04:02.265012+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: audit 2023-10-09T06:04:03.261084+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: audit 2023-10-09T06:04:03.304814+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: audit 2023-10-09T06:04:03.308600+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: audit 2023-10-09T06:04:03.310440+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[27513]: audit 2023-10-09T06:04:03.317987+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: cluster 2023-10-09T06:04:02.265012+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: audit 2023-10-09T06:04:03.261084+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: audit 2023-10-09T06:04:03.304814+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:03.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: audit 2023-10-09T06:04:03.308600+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:03.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: audit 2023-10-09T06:04:03.310440+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:03.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:03 smithi125 bash[33919]: audit 2023-10-09T06:04:03.317987+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: cluster 2023-10-09T06:04:02.265012+0000 mgr.y (mgr.14430) 174 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 65 KiB/s wr, 38 op/s 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: audit 2023-10-09T06:04:03.261084+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:04.906 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: audit 2023-10-09T06:04:03.304814+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:04.907 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: audit 2023-10-09T06:04:03.308600+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:04.907 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: audit 2023-10-09T06:04:03.310440+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:04.907 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:03 smithi155 bash[27817]: audit 2023-10-09T06:04:03.317987+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:04.908 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:04 smithi155 bash[27817]: audit 2023-10-09T06:04:03.252889+0000 mgr.y (mgr.14430) 175 : audit [DBG] from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:05.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:04 smithi155 bash[27817]: cephadm 2023-10-09T06:04:03.254361+0000 mgr.y (mgr.14430) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:04 smithi155 bash[27817]: cephadm 2023-10-09T06:04:03.323398+0000 mgr.y (mgr.14430) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.005 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:05.054 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:04:05.054 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:04:05.054 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img2 2023-10-09T06:04:05.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[27513]: audit 2023-10-09T06:04:03.252889+0000 mgr.y (mgr.14430) 175 : audit [DBG] from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:05.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[27513]: cephadm 2023-10-09T06:04:03.254361+0000 mgr.y (mgr.14430) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[27513]: cephadm 2023-10-09T06:04:03.323398+0000 mgr.y (mgr.14430) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[33919]: audit 2023-10-09T06:04:03.252889+0000 mgr.y (mgr.14430) 175 : audit [DBG] from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:05.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[33919]: cephadm 2023-10-09T06:04:03.254361+0000 mgr.y (mgr.14430) 176 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:04 smithi125 bash[33919]: cephadm 2023-10-09T06:04:03.323398+0000 mgr.y (mgr.14430) 177 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T06:04:05.055 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img3 2023-10-09T06:04:05.405 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img2 /tmp/rbd_import_export_66110/img3 2023-10-09T06:04:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:05 smithi125 bash[27513]: cluster 2023-10-09T06:04:04.265511+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 57 KiB/s wr, 33 op/s 2023-10-09T06:04:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:05 smithi125 bash[33919]: cluster 2023-10-09T06:04:04.265511+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 57 KiB/s wr, 33 op/s 2023-10-09T06:04:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:05 smithi155 bash[27817]: cluster 2023-10-09T06:04:04.265511+0000 mgr.y (mgr.14430) 178 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 57 KiB/s wr, 33 op/s 2023-10-09T06:04:07.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd help export 2023-10-09T06:04:07.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -q export-format 2023-10-09T06:04:07.489 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_66110/img bs=1k count=10 seek=100 2023-10-09T06:04:07.542 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T06:04:07.542 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T06:04:07.542 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000177888 s, 57.6 MB/s 2023-10-09T06:04:07.542 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:04:07.628 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:07.629 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:07.632 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap 2023-10-09T06:04:07.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:07 smithi125 bash[27513]: cluster 2023-10-09T06:04:06.266451+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:07 smithi125 bash[33919]: cluster 2023-10-09T06:04:06.266451+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:07 smithi155 bash[27817]: cluster 2023-10-09T06:04:06.266451+0000 mgr.y (mgr.14430) 179 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:08.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:04:08.449 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:04:08.459 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T06:04:08.524 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta list testimg 2023-10-09T06:04:08.578 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T06:04:08.578 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:04:08.578 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:04:08.578 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 2023-10-09T06:04:08.578 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:08.633 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:08.637 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_66110/img_v2 testimg_import 2023-10-09T06:04:09.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:08] "GET /metrics HTTP/1.1" 200 191545 "" "Prometheus/2.33.4" 2023-10-09T06:04:09.468 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:04:09.473 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:04:09.515 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T06:04:09.515 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: id: 3a2c2ca8bb4 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.3a2c2ca8bb4 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.516 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: id: 3a2c2ca8bb4 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.3a2c2ca8bb4 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T06:04:09.569 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T06:04:09.570 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.570 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.570 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 06:04:08 2023 2023-10-09T06:04:09.570 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T06:04:09.573 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta list testimg_import 2023-10-09T06:04:09.631 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:04:09.632 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 ] 2023-10-09T06:04:09.633 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import /tmp/rbd_import_export_66110/img_import 2023-10-09T06:04:09.685 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:04:09.690 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img_import 2023-10-09T06:04:09.690 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/img /tmp/rbd_import_export_66110/img_import 2023-10-09T06:04:09.691 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/img --format %b 2023-10-09T06:04:09.693 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=24 2023-10-09T06:04:09.693 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/img_import --format %b 2023-10-09T06:04:09.695 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T06:04:09.695 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T06:04:09.695 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T06:04:09.695 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T06:04:09.695 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_66110/img_snap 2023-10-09T06:04:09.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:09 smithi125 bash[33919]: cluster 2023-10-09T06:04:08.267076+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:09.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:09 smithi125 bash[33919]: cluster 2023-10-09T06:04:08.431174+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T06:04:09.738 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:09 smithi125 bash[27513]: cluster 2023-10-09T06:04:08.267076+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:09.739 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:09 smithi125 bash[27513]: cluster 2023-10-09T06:04:08.431174+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T06:04:09.745 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:04:09.750 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_66110/img_snap_import 2023-10-09T06:04:09.804 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:04:09.808 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/img_snap /tmp/rbd_import_export_66110/img_snap_import 2023-10-09T06:04:09.808 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/img_snap /tmp/rbd_import_export_66110/img_snap_import 2023-10-09T06:04:09.809 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/img_snap --format %b 2023-10-09T06:04:09.810 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=224 2023-10-09T06:04:09.810 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/img_snap_import --format %b 2023-10-09T06:04:09.811 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T06:04:09.811 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:09.811 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:09.811 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:04:09.812 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:09.812 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_import 2023-10-09T06:04:09.812 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_snap 2023-10-09T06:04:09.813 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_snap_import 2023-10-09T06:04:09.814 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T06:04:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:09 smithi155 bash[27817]: cluster 2023-10-09T06:04:08.267076+0000 mgr.y (mgr.14430) 180 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 252 KiB/s rd, 82 KiB/s wr, 73 op/s 2023-10-09T06:04:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:09 smithi155 bash[27817]: cluster 2023-10-09T06:04:08.431174+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-09T06:04:10.463 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T06:04:10.472 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:04:10.617 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:10.622 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg@snap 2023-10-09T06:04:10.751 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:10 smithi125 bash[27513]: cluster 2023-10-09T06:04:09.440618+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T06:04:10.751 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:10 smithi125 bash[33919]: cluster 2023-10-09T06:04:09.440618+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T06:04:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:10 smithi155 bash[27817]: cluster 2023-10-09T06:04:09.440618+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-09T06:04:11.472 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T06:04:11.482 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:04:11.633 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:11.638 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --order 20 /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:04:11.714 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:11.715 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:11.719 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:11.771 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:11.775 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_66110/img_v2 testimg_import 2023-10-09T06:04:11.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:11 smithi125 bash[27513]: cluster 2023-10-09T06:04:10.267677+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 236 KiB/s rd, 73 KiB/s wr, 144 op/s 2023-10-09T06:04:11.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:11 smithi125 bash[27513]: cluster 2023-10-09T06:04:10.455400+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T06:04:11.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:11 smithi125 bash[33919]: cluster 2023-10-09T06:04:10.267677+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 236 KiB/s rd, 73 KiB/s wr, 144 op/s 2023-10-09T06:04:11.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:11 smithi125 bash[33919]: cluster 2023-10-09T06:04:10.455400+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T06:04:11.875 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:04:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:11 smithi155 bash[27817]: cluster 2023-10-09T06:04:10.267677+0000 mgr.y (mgr.14430) 181 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 236 KiB/s rd, 73 KiB/s wr, 144 op/s 2023-10-09T06:04:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:11 smithi155 bash[27817]: cluster 2023-10-09T06:04:10.455400+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-09T06:04:11.881 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:04:11.881 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep order 2023-10-09T06:04:11.881 INFO:tasks.workunit.client.0.smithi125.stderr:+ + awk {print $2} 2023-10-09T06:04:11.881 INFO:tasks.workunit.client.0.smithi125.stderr:grep 20 2023-10-09T06:04:11.933 INFO:tasks.workunit.client.0.smithi125.stdout:20 2023-10-09T06:04:11.934 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:11.934 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:04:12.066 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:12.071 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:04:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:12.207 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:12.212 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:04:12.277 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:12.282 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg 2023-10-09T06:04:12.282 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep features 2023-10-09T06:04:12.333 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T06:04:12.334 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: 2023-10-09T06:04:12.334 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (3m) 19s ago 4m 12.7M - 0.23.0 ba2b418f427c afb24507e7be 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (3m) 20s ago 3m 43.3M - 8.3.5 dad864ee21e9 956b3243cbf0 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (2m) 19s ago 2m 13.0M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (2m) 20s ago 2m 11.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443 running (11m) 20s ago 11m 374M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 9f09c75e6322 2023-10-09T06:04:12.345 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:9283 running (13m) 19s ago 13m 427M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 8a68e19a35c1 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (13m) 19s ago 13m 51.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 520280efd67b 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (11m) 20s ago 11m 43.3M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 920321cd9092 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (11m) 19s ago 11m 44.3M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7b9a79d51662 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (4m) 19s ago 4m 9600k - 1.3.1 1dbe0e931976 7608647a2e14 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (4m) 20s ago 4m 9679k - 1.3.1 1dbe0e931976 5a8642b6efb5 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (10m) 19s ago 10m 38.3M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4276c724754b 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (9m) 19s ago 9m 38.4M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 67dcf3157f53 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (9m) 19s ago 9m 36.9M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 05b5c73536d5 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (8m) 19s ago 8m 37.5M 4060M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6d440ba42e37 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (7m) 20s ago 7m 38.3M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:04:12.346 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (6m) 20s ago 6m 37.6M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:04:12.347 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (6m) 20s ago 6m 37.9M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:04:12.347 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (5m) 20s ago 5m 37.3M 4316M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:04:12.347 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (3m) 20s ago 4m 50.9M - 2.33.4 514e6a882f6e d1d6ef0f2cf1 2023-10-09T06:04:12.387 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:12.392 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_66110/img_v2 testimg_import 2023-10-09T06:04:12.468 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:04:12.473 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:04:12.473 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep features 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ features: layering 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: != features: layering 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ] 2023-10-09T06:04:12.517 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:12.518 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:04:12.619 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:12.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:04:12.734 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:12.739 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:04:12.825 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:12.829 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:12.880 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:12.885 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_66110/img_v2 testimg_import 2023-10-09T06:04:12.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:12 smithi125 bash[27513]: cluster 2023-10-09T06:04:11.464096+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T06:04:12.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:12 smithi125 bash[33919]: cluster 2023-10-09T06:04:11.464096+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T06:04:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:12 smithi155 bash[27817]: cluster 2023-10-09T06:04:11.464096+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-09T06:04:12.978 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:04:12.984 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:04:12.984 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep stripe unit 2023-10-09T06:04:12.984 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T06:04:13.030 INFO:tasks.workunit.client.0.smithi125.stdout: stripe unit: 4 KiB 2023-10-09T06:04:13.031 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:04:13.031 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep stripe count 2023-10-09T06:04:13.031 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk+ {print $3} 2023-10-09T06:04:13.031 INFO:tasks.workunit.client.0.smithi125.stderr:grep 1000 2023-10-09T06:04:13.076 INFO:tasks.workunit.client.0.smithi125.stdout:1000 2023-10-09T06:04:13.076 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/img_v2 2023-10-09T06:04:13.076 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:04:13.211 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:04:13.215 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:04:13.355 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:04:13.360 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_66110/img testimg 2023-10-09T06:04:13.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T06:04:13.449 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:13.451 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap1 2023-10-09T06:04:13.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:13 smithi125 bash[33919]: cluster 2023-10-09T06:04:12.268125+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 163 op/s 2023-10-09T06:04:13.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:13 smithi125 bash[33919]: audit 2023-10-09T06:04:12.325337+0000 mgr.y (mgr.14430) 183 : audit [DBG] from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:13 smithi125 bash[27513]: cluster 2023-10-09T06:04:12.268125+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 163 op/s 2023-10-09T06:04:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:13 smithi125 bash[27513]: audit 2023-10-09T06:04:12.325337+0000 mgr.y (mgr.14430) 183 : audit [DBG] from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:13 smithi155 bash[27817]: cluster 2023-10-09T06:04:12.268125+0000 mgr.y (mgr.14430) 182 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 163 op/s 2023-10-09T06:04:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:13 smithi155 bash[27817]: audit 2023-10-09T06:04:12.325337+0000 mgr.y (mgr.14430) 183 : audit [DBG] from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:14.499 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:04:14.510 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap2 2023-10-09T06:04:15.507 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:04:15.517 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap protect testimg@snap2 2023-10-09T06:04:15.573 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_66110/snap_protect 2023-10-09T06:04:15.624 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:15.630 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_66110/snap_protect testimg_import 2023-10-09T06:04:15.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:15 smithi125 bash[27513]: cluster 2023-10-09T06:04:14.268718+0000 mgr.y (mgr.14430) 184 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.2 KiB/s wr, 112 op/s 2023-10-09T06:04:15.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:15 smithi125 bash[27513]: cluster 2023-10-09T06:04:14.482547+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T06:04:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:15 smithi125 bash[33919]: cluster 2023-10-09T06:04:14.268718+0000 mgr.y (mgr.14430) 184 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.2 KiB/s wr, 112 op/s 2023-10-09T06:04:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:15 smithi125 bash[33919]: cluster 2023-10-09T06:04:14.482547+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T06:04:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:15 smithi155 bash[27817]: cluster 2023-10-09T06:04:14.268718+0000 mgr.y (mgr.14430) 184 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.2 KiB/s wr, 112 op/s 2023-10-09T06:04:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:15 smithi155 bash[27817]: cluster 2023-10-09T06:04:14.482547+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-09T06:04:16.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:16 smithi125 bash[27513]: cluster 2023-10-09T06:04:15.492251+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T06:04:16.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:16 smithi125 bash[33919]: cluster 2023-10-09T06:04:15.492251+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T06:04:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:16 smithi155 bash[27817]: cluster 2023-10-09T06:04:15.492251+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-09T06:04:17.113 INFO:teuthology.orchestra.run.smithi125.stdout:wait for minority of mons to upgrade 2023-10-09T06:04:17.535 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:04:17.539 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap1 2023-10-09T06:04:17.539 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep protected: False 2023-10-09T06:04:17.590 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T06:04:17.590 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap2 2023-10-09T06:04:17.590 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep protected: True 2023-10-09T06:04:17.640 INFO:tasks.workunit.client.0.smithi125.stdout: protected: True 2023-10-09T06:04:17.640 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/snap_protect 2023-10-09T06:04:17.641 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T06:04:17.703 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T06:04:17.760 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg 2023-10-09T06:04:17.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:17 smithi125 bash[27513]: cluster 2023-10-09T06:04:16.269421+0000 mgr.y (mgr.14430) 185 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 262 KiB/s rd, 31 KiB/s wr, 353 op/s 2023-10-09T06:04:17.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:17 smithi125 bash[27513]: cluster 2023-10-09T06:04:16.499280+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T06:04:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:17 smithi125 bash[33919]: cluster 2023-10-09T06:04:16.269421+0000 mgr.y (mgr.14430) 185 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 262 KiB/s rd, 31 KiB/s wr, 353 op/s 2023-10-09T06:04:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:17 smithi125 bash[33919]: cluster 2023-10-09T06:04:16.499280+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T06:04:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:17 smithi155 bash[27817]: cluster 2023-10-09T06:04:16.269421+0000 mgr.y (mgr.14430) 185 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 262 KiB/s rd, 31 KiB/s wr, 353 op/s 2023-10-09T06:04:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:17 smithi155 bash[27817]: cluster 2023-10-09T06:04:16.499280+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-09T06:04:18.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T06:04:18.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:18 smithi125 bash[27513]: cluster 2023-10-09T06:04:17.500188+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T06:04:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:18 smithi125 bash[33919]: cluster 2023-10-09T06:04:17.500188+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T06:04:18.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:18] "GET /metrics HTTP/1.1" 200 192066 "" "Prometheus/2.33.4" 2023-10-09T06:04:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:18 smithi155 bash[27817]: cluster 2023-10-09T06:04:17.500188+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-09T06:04:19.533 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:04:19.543 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg_import 2023-10-09T06:04:19.780 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:19 smithi125 bash[27513]: cluster 2023-10-09T06:04:18.270016+0000 mgr.y (mgr.14430) 186 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 45 KiB/s wr, 513 op/s 2023-10-09T06:04:19.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:19 smithi125 bash[27513]: cluster 2023-10-09T06:04:18.515322+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T06:04:19.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:19 smithi125 bash[33919]: cluster 2023-10-09T06:04:18.270016+0000 mgr.y (mgr.14430) 186 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 45 KiB/s wr, 513 op/s 2023-10-09T06:04:19.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:19 smithi125 bash[33919]: cluster 2023-10-09T06:04:18.515322+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T06:04:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:19 smithi155 bash[27817]: cluster 2023-10-09T06:04:18.270016+0000 mgr.y (mgr.14430) 186 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 112 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 381 KiB/s rd, 45 KiB/s wr, 513 op/s 2023-10-09T06:04:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:19 smithi155 bash[27817]: cluster 2023-10-09T06:04:18.515322+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-09T06:04:20.782 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:20 smithi125 bash[33919]: cluster 2023-10-09T06:04:19.523530+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:04:20.782 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:20 smithi125 bash[33919]: cluster 2023-10-09T06:04:20.412224+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:04:20.783 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:20 smithi125 bash[27513]: cluster 2023-10-09T06:04:19.523530+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:04:20.783 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:20 smithi125 bash[27513]: cluster 2023-10-09T06:04:20.412224+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:04:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:20 smithi155 bash[27817]: cluster 2023-10-09T06:04:19.523530+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:04:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:20 smithi155 bash[27817]: cluster 2023-10-09T06:04:20.412224+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:04:21.433 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:04:21.442 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:04:21.597 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:21.600 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:04:21.744 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:21.749 INFO:tasks.workunit.client.0.smithi125.stderr:+ tiered=0 2023-10-09T06:04:21.749 INFO:tasks.workunit.client.0.smithi125.stderr:+ ceph osd dump 2023-10-09T06:04:21.749 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep ^pool 2023-10-09T06:04:21.749 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'rbd' 2023-10-09T06:04:21.749 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep tier 2023-10-09T06:04:21.866 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:21 smithi125 bash[27513]: cluster 2023-10-09T06:04:20.271064+0000 mgr.y (mgr.14430) 187 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:21.866 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:21 smithi125 bash[27513]: cluster 2023-10-09T06:04:21.425977+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:04:21.867 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:21 smithi125 bash[33919]: cluster 2023-10-09T06:04:20.271064+0000 mgr.y (mgr.14430) 187 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:21.867 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:21 smithi125 bash[33919]: cluster 2023-10-09T06:04:21.425977+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:04:21.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:21 smithi155 bash[27817]: cluster 2023-10-09T06:04:20.271064+0000 mgr.y (mgr.14430) 187 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:21.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:21 smithi155 bash[27817]: cluster 2023-10-09T06:04:21.425977+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:04:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:22.167 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_66110/sparse1 2023-10-09T06:04:22.172 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:04:22.172 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:04:22.172 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00372597 s, 281 MB/s 2023-10-09T06:04:22.172 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_66110/sparse2 2023-10-09T06:04:22.182 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:04:22.182 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:04:22.182 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00878075 s, 119 MB/s 2023-10-09T06:04:22.182 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_66110/sparse2 -s 2M 2023-10-09T06:04:22.188 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:04:22.240 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:04:22.240 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:04:22.245 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:04:22.245 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_66110/sparse1 2023-10-09T06:04:22.342 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:04:22.342 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:22.343 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:22.346 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:04:22.347 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grep sparse1 2023-10-09T06:04:22.347 INFO:tasks.workunit.client.0.smithi125.stderr:grep -Ei (2 MiB|2048k) 2023-10-09T06:04:22.394 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 2 MiB 2 2023-10-09T06:04:22.395 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse1 2023-10-09T06:04:22.395 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:04:22.395 INFO:tasks.workunit.client.0.smithi125.stderr:+ + rbd infogrep sparse1 block_name_prefix 2023-10-09T06:04:22.395 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:04:22.395 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:22.449 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.620670c09f34 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.620670c09f34 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse1 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:+ + sed -e s/rbd_data.620670c09f34\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:sort -u 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd+ info sparse1 2023-10-09T06:04:22.450 INFO:tasks.workunit.client.0.smithi125.stderr:grep data_pool: 2023-10-09T06:04:22.451 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:22.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:22.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:22.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:22.500 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:22.500 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:22.546 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:04:22.546 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:04:22.546 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:04:22.546 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:22.593 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:22.598 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:22.598 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:22.601 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse1 --format %b 2023-10-09T06:04:22.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:04:22.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse1.out --format %b 2023-10-09T06:04:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:04:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:04:22.604 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:22.605 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:04:22.737 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:04:22.741 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:04:22.795 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:04:22.795 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:04:22.799 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:04:22.799 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_66110/sparse2 2023-10-09T06:04:22.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[27513]: audit 2023-10-09T06:04:22.032931+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[27513]: audit 2023-10-09T06:04:22.149850+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[27513]: audit 2023-10-09T06:04:22.453362+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.125:0/3659379706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:22.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[33919]: audit 2023-10-09T06:04:22.032931+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[33919]: audit 2023-10-09T06:04:22.149850+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:22 smithi125 bash[33919]: audit 2023-10-09T06:04:22.453362+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.125:0/3659379706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:22.892 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:04:22.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:22 smithi155 bash[27817]: audit 2023-10-09T06:04:22.032931+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:22 smithi155 bash[27817]: audit 2023-10-09T06:04:22.149850+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.125:0/655096628' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:04:22.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:22 smithi155 bash[27817]: audit 2023-10-09T06:04:22.453362+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.125:0/3659379706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:22.894 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:22.894 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:22.897 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:04:22.897 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse2 2023-10-09T06:04:22.897 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T06:04:22.952 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 2 MiB 2 2023-10-09T06:04:22.953 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse2 2023-10-09T06:04:22.953 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:04:22.954 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd+ grep block_name_prefix 2023-10-09T06:04:22.954 INFO:tasks.workunit.client.0.smithi125.stderr: info+ awk {print $NF;} 2023-10-09T06:04:22.954 INFO:tasks.workunit.client.0.smithi125.stderr: sparse2 2023-10-09T06:04:23.007 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3b857d085eaa 2023-10-09T06:04:23.007 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.3b857d085eaa 2023-10-09T06:04:23.007 INFO:tasks.workunit.client.0.smithi125.stderr:+ + sed -e s/rbd_data.3b857d085eaa\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:23.007 INFO:tasks.workunit.client.0.smithi125.stderr:get_image_data_pool sparse2 2023-10-09T06:04:23.007 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:04:23.008 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:04:23.008 INFO:tasks.workunit.client.0.smithi125.stderr:+ + rbd info sparse2 2023-10-09T06:04:23.008 INFO:tasks.workunit.client.0.smithi125.stderr:grep data_pool: 2023-10-09T06:04:23.008 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:23.064 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:23.064 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:23.064 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:23.064 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:23.064 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:23.108 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:04:23.108 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2023-10-09T06:04:23.108 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T06:04:23.108 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:23.159 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:23.164 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:23.164 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:23.167 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse2 --format %b 2023-10-09T06:04:23.168 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:04:23.168 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse2.out --format %b 2023-10-09T06:04:23.169 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:04:23.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:23.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:23.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:04:23.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:23.170 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:04:23.311 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:04:23.316 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_66110/sparse1 -s 10M 2023-10-09T06:04:23.317 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T06:04:23.460 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:23.461 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:23.461 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:23.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:04:23.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse1 2023-10-09T06:04:23.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-09T06:04:23.526 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 10 MiB 2 2023-10-09T06:04:23.528 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse1 2023-10-09T06:04:23.528 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:04:23.528 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:04:23.528 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:04:23.528 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:23.583 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3bb2d2dccad2 2023-10-09T06:04:23.583 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.3bb2d2dccad2 2023-10-09T06:04:23.583 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse1 2023-10-09T06:04:23.583 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:04:23.583 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.3bb2d2dccad2\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:23.584 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:04:23.584 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:04:23.584 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:04:23.587 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:23.634 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:23.634 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:23.635 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:23.635 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:23.635 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:23.685 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:04:23.685 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:04:23.685 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:04:23.685 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:23.737 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:23.739 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:23.739 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/sparse1 /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:23.751 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse1 --format %b 2023-10-09T06:04:23.752 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:04:23.752 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse1.out --format %b 2023-10-09T06:04:23.753 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:04:23.753 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:23.754 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:23.754 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:04:23.754 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse1.out 2023-10-09T06:04:23.754 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:04:23.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:23 smithi125 bash[33919]: cluster 2023-10-09T06:04:22.271535+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:23.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:23 smithi125 bash[27513]: cluster 2023-10-09T06:04:22.271535+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:23 smithi155 bash[27817]: cluster 2023-10-09T06:04:22.271535+0000 mgr.y (mgr.14430) 188 : cluster [DBG] pgmap v178: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-10-09T06:04:23.895 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:04:23.899 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_66110/sparse2 oflag=append conv=notrunc 2023-10-09T06:04:23.907 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:04:23.907 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:04:23.908 INFO:tasks.workunit.client.0.smithi125.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00777696 s, 270 MB/s 2023-10-09T06:04:23.908 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T06:04:24.024 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:24.025 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:24.025 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:24.029 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:04:24.029 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse2 2023-10-09T06:04:24.029 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T06:04:24.084 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 4 MiB 2 2023-10-09T06:04:24.085 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse2 2023-10-09T06:04:24.085 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:04:24.086 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:04:24.086 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:04:24.086 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:24.139 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.62932bbdcb31 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.62932bbdcb31 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse2 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed+ sort -u 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr: -e s/rbd_data.62932bbdcb31\.// -e+ grep data_pool: 2023-10-09T06:04:24.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:24.142 INFO:tasks.workunit.client.0.smithi125.stderr: s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:24.142 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:04:24.198 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:24.198 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:24.198 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:24.198 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:24.198 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:24.241 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:04:24.241 INFO:tasks.workunit.client.0.smithi125.stderr:2 2023-10-09T06:04:24.241 INFO:tasks.workunit.client.0.smithi125.stderr:3 2023-10-09T06:04:24.242 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2 3 2023-10-09T06:04:24.242 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T06:04:24.242 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:24.296 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T06:04:24.302 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_66110/sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:24.302 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_66110/sparse2 /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:24.306 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse2 --format %b 2023-10-09T06:04:24.307 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=6144 2023-10-09T06:04:24.307 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse2.out --format %b 2023-10-09T06:04:24.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=6144 2023-10-09T06:04:24.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:24.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:04:24.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:04:24.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse2.out 2023-10-09T06:04:24.310 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:04:24.460 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T06:04:24.465 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T06:04:24.466 INFO:tasks.workunit.client.0.smithi125.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T06:04:24.466 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_66110/sparse1 sparse 2023-10-09T06:04:24.565 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T06:04:24.566 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:24.567 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:04:24.571 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:04:24.571 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:24.571 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:24.571 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:04:24.571 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:24.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3c0ca509745a 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ + get_image_data_pool sparsegrep 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr: rbd_data.3c0ca509745a 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.3c0ca509745a\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep+ awk -F: { print $NF } 2023-10-09T06:04:24.624 INFO:tasks.workunit.client.0.smithi125.stderr: data_pool: 2023-10-09T06:04:24.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:24.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:24.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:24.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:24.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:24.722 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:04:24.722 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:04:24.723 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:04:24.723 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:04:24.856 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:04:24.860 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T06:04:24.860 INFO:tasks.workunit.client.0.smithi125.stdout:zeros import through stdin to sparse image 2023-10-09T06:04:24.861 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T06:04:24.861 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format - sparse 2023-10-09T06:04:24.928 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records in 2023-10-09T06:04:24.928 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records out 2023-10-09T06:04:24.928 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0665017 s, 63.1 MB/s 2023-10-09T06:04:24.947 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:04:24.951 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:24.955 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:04:24.955 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:24.956 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:24.956 INFO:tasks.workunit.client.0.smithi125.stderr:+ + awk {print $NF;} 2023-10-09T06:04:24.956 INFO:tasks.workunit.client.0.smithi125.stderr:grep block_name_prefix 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3c33f832b0b3 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.3c33f832b0b3 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.3c33f832b0b3\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:04:25.003 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:25.004 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:25.004 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:04:25.004 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:25.049 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:25.049 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:25.049 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:25.049 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:25.050 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:25.094 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects= 2023-10-09T06:04:25.094 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 2023-10-09T06:04:25.094 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T06:04:25.094 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:04:25.233 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:25.238 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo zeros export to sparse file 2023-10-09T06:04:25.238 INFO:tasks.workunit.client.0.smithi125.stdout:zeros export to sparse file 2023-10-09T06:04:25.238 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T06:04:25.294 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:04:25.299 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:25.299 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:04:25.299 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:25.354 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3c54b71566f7 2023-10-09T06:04:25.354 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T06:04:25.354 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:04:25.354 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:25.355 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:25.355 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:04:25.356 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:25.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:25.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:25.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:25.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:25.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados -p rbd put rbd_data.3c54b71566f7.000000000000 - 2023-10-09T06:04:25.443 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:04:25.443 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:04:25.443 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0879226 s, 47.7 MB/s 2023-10-09T06:04:25.469 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:04:25.469 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:25.469 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:25.469 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:04:25.469 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:04:25.517 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.3c54b71566f7 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.3c54b71566f7 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.3c54b71566f7\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:04:25.519 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:04:25.520 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:04:25.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:04:25.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:04:25.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:04:25.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:04:25.577 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:04:25.626 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:04:25.626 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2023-10-09T06:04:25.627 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T06:04:25.627 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse 2023-10-09T06:04:25.627 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_66110/sparse': No such file or directory 2023-10-09T06:04:25.628 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:04:25.628 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse /tmp/rbd_import_export_66110/sparse 2023-10-09T06:04:25.679 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:04:25.684 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_66110/sparse --format=%b 2023-10-09T06:04:25.686 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 = 0 ] 2023-10-09T06:04:25.686 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:04:25.826 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:04:25.834 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_66110/sparse /tmp/rbd_import_export_66110/sparse1 /tmp/rbd_import_export_66110/sparse2 /tmp/rbd_import_export_66110/sparse3 2023-10-09T06:04:25.835 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_66110/sparse3': No such file or directory 2023-10-09T06:04:25.835 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:04:25.836 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo OK 2023-10-09T06:04:25.836 INFO:tasks.workunit.client.0.smithi125.stdout:OK 2023-10-09T06:04:25.838 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_66110 2023-10-09T06:04:25.838 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:04:25.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:04:25.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:25 smithi125 bash[33919]: cluster 2023-10-09T06:04:24.271996+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 8.3 KiB/s wr, 119 op/s 2023-10-09T06:04:25.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:25 smithi125 bash[33919]: audit 2023-10-09T06:04:24.963231+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.125:0/2297338169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:25.849 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:25 smithi125 bash[27513]: cluster 2023-10-09T06:04:24.271996+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 8.3 KiB/s wr, 119 op/s 2023-10-09T06:04:25.850 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:25 smithi125 bash[27513]: audit 2023-10-09T06:04:24.963231+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.125:0/2297338169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:25.852 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T06:04:25.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:04:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:25 smithi155 bash[27817]: cluster 2023-10-09T06:04:24.271996+0000 mgr.y (mgr.14430) 189 : cluster [DBG] pgmap v179: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 8.3 KiB/s wr, 119 op/s 2023-10-09T06:04:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:25 smithi155 bash[27817]: audit 2023-10-09T06:04:24.963231+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.125:0/2297338169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:26.148 DEBUG:teuthology.parallel:result is None 2023-10-09T06:04:26.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:04:26.161 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:04:26.161 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:04:26.209 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:04:26.210 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:04:26.210 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T06:04:26.210 DEBUG:teuthology.parallel:result is None 2023-10-09T06:04:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:27 smithi125 bash[27513]: cluster 2023-10-09T06:04:26.272864+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.1 MiB/s wr, 449 op/s 2023-10-09T06:04:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:27 smithi125 bash[27513]: audit 2023-10-09T06:04:27.440913+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.125:0/3079667147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:27 smithi125 bash[33919]: cluster 2023-10-09T06:04:26.272864+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.1 MiB/s wr, 449 op/s 2023-10-09T06:04:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:27 smithi125 bash[33919]: audit 2023-10-09T06:04:27.440913+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.125:0/3079667147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:27 smithi155 bash[27817]: cluster 2023-10-09T06:04:26.272864+0000 mgr.y (mgr.14430) 190 : cluster [DBG] pgmap v180: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.1 MiB/s wr, 449 op/s 2023-10-09T06:04:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:27 smithi155 bash[27817]: audit 2023-10-09T06:04:27.440913+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.125:0/3079667147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:29.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:28] "GET /metrics HTTP/1.1" 200 192204 "" "Prometheus/2.33.4" 2023-10-09T06:04:29.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:29 smithi125 bash[27513]: cluster 2023-10-09T06:04:28.273338+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 987 KiB/s rd, 909 KiB/s wr, 293 op/s 2023-10-09T06:04:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:29 smithi125 bash[33919]: cluster 2023-10-09T06:04:28.273338+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 987 KiB/s rd, 909 KiB/s wr, 293 op/s 2023-10-09T06:04:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:29 smithi155 bash[27817]: cluster 2023-10-09T06:04:28.273338+0000 mgr.y (mgr.14430) 191 : cluster [DBG] pgmap v181: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 987 KiB/s rd, 909 KiB/s wr, 293 op/s 2023-10-09T06:04:30.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:30 smithi125 bash[27513]: audit 2023-10-09T06:04:29.917273+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.125:0/1447581406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:30.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:30 smithi125 bash[27513]: cluster 2023-10-09T06:04:30.415968+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:04:30.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:30 smithi125 bash[33919]: audit 2023-10-09T06:04:29.917273+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.125:0/1447581406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:30.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:30 smithi125 bash[33919]: cluster 2023-10-09T06:04:30.415968+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:04:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:30 smithi155 bash[27817]: audit 2023-10-09T06:04:29.917273+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.125:0/1447581406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:30 smithi155 bash[27817]: cluster 2023-10-09T06:04:30.415968+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:04:31.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:31 smithi155 bash[27817]: cluster 2023-10-09T06:04:30.274274+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s 2023-10-09T06:04:31.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:31 smithi125 bash[27513]: cluster 2023-10-09T06:04:30.274274+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s 2023-10-09T06:04:31.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:31 smithi125 bash[33919]: cluster 2023-10-09T06:04:30.274274+0000 mgr.y (mgr.14430) 192 : cluster [DBG] pgmap v182: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 823 KiB/s rd, 1.1 MiB/s wr, 267 op/s 2023-10-09T06:04:32.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:31 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:32.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:32 smithi125 bash[27513]: audit 2023-10-09T06:04:32.382718+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.125:0/1962394315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:32.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:32 smithi125 bash[33919]: audit 2023-10-09T06:04:32.382718+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.125:0/1962394315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:32.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:32 smithi155 bash[27817]: audit 2023-10-09T06:04:32.382718+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.125:0/1962394315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:33.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:33 smithi125 bash[27513]: cluster 2023-10-09T06:04:32.274730+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:33.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:33 smithi125 bash[33919]: cluster 2023-10-09T06:04:32.274730+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:33 smithi155 bash[27817]: cluster 2023-10-09T06:04:32.274730+0000 mgr.y (mgr.14430) 193 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:35.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:35 smithi125 bash[27513]: cluster 2023-10-09T06:04:34.275354+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:35.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:35 smithi125 bash[27513]: audit 2023-10-09T06:04:34.799989+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.125:0/1648458788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:35 smithi125 bash[33919]: cluster 2023-10-09T06:04:34.275354+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:35 smithi125 bash[33919]: audit 2023-10-09T06:04:34.799989+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.125:0/1648458788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:35 smithi155 bash[27817]: cluster 2023-10-09T06:04:34.275354+0000 mgr.y (mgr.14430) 194 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 811 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2023-10-09T06:04:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:35 smithi155 bash[27817]: audit 2023-10-09T06:04:34.799989+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.125:0/1648458788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:37 smithi125 bash[27513]: cluster 2023-10-09T06:04:36.276480+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:37 smithi125 bash[27513]: audit 2023-10-09T06:04:37.197255+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.125:0/3481137809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:37 smithi125 bash[33919]: cluster 2023-10-09T06:04:36.276480+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:37 smithi125 bash[33919]: audit 2023-10-09T06:04:37.197255+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.125:0/3481137809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:37 smithi155 bash[27817]: cluster 2023-10-09T06:04:36.276480+0000 mgr.y (mgr.14430) 195 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:37 smithi155 bash[27817]: audit 2023-10-09T06:04:37.197255+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.125:0/3481137809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:39.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:38 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:38] "GET /metrics HTTP/1.1" 200 192204 "" "Prometheus/2.33.4" 2023-10-09T06:04:39.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:39 smithi125 bash[27513]: cluster 2023-10-09T06:04:38.277090+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:39 smithi125 bash[33919]: cluster 2023-10-09T06:04:38.277090+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:39 smithi155 bash[27817]: cluster 2023-10-09T06:04:38.277090+0000 mgr.y (mgr.14430) 196 : cluster [DBG] pgmap v187: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 410 KiB/s wr, 28 op/s 2023-10-09T06:04:40.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:40 smithi125 bash[33919]: audit 2023-10-09T06:04:39.617619+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.125:0/1983088017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:40.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:40 smithi125 bash[27513]: audit 2023-10-09T06:04:39.617619+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.125:0/1983088017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:40 smithi155 bash[27817]: audit 2023-10-09T06:04:39.617619+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.125:0/1983088017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:41.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:41 smithi155 bash[27817]: cluster 2023-10-09T06:04:40.278134+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:41.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:41 smithi125 bash[33919]: cluster 2023-10-09T06:04:40.278134+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:41.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:41 smithi125 bash[27513]: cluster 2023-10-09T06:04:40.278134+0000 mgr.y (mgr.14430) 197 : cluster [DBG] pgmap v188: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:42.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:41 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:42.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:42 smithi125 bash[27513]: audit 2023-10-09T06:04:42.018930+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.125:0/2072338089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:42.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:42 smithi125 bash[33919]: audit 2023-10-09T06:04:42.018930+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.125:0/2072338089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:42 smithi155 bash[27817]: audit 2023-10-09T06:04:42.018930+0000 mon.a (mon.0) 715 : audit [DBG] from='client.? 172.21.15.125:0/2072338089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:43.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:43 smithi125 bash[27513]: cluster 2023-10-09T06:04:42.278774+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:43 smithi125 bash[33919]: cluster 2023-10-09T06:04:42.278774+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:43 smithi155 bash[27817]: cluster 2023-10-09T06:04:42.278774+0000 mgr.y (mgr.14430) 198 : cluster [DBG] pgmap v189: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:44.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:44 smithi125 bash[27513]: audit 2023-10-09T06:04:44.420206+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.125:0/3270556491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:44 smithi125 bash[33919]: audit 2023-10-09T06:04:44.420206+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.125:0/3270556491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:44 smithi155 bash[27817]: audit 2023-10-09T06:04:44.420206+0000 mon.a (mon.0) 716 : audit [DBG] from='client.? 172.21.15.125:0/3270556491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[27513]: cluster 2023-10-09T06:04:44.279159+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[27513]: audit 2023-10-09T06:04:45.010533+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[27513]: audit 2023-10-09T06:04:45.015385+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[27513]: audit 2023-10-09T06:04:45.022783+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[33919]: cluster 2023-10-09T06:04:44.279159+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[33919]: audit 2023-10-09T06:04:45.010533+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[33919]: audit 2023-10-09T06:04:45.015385+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:45 smithi125 bash[33919]: audit 2023-10-09T06:04:45.022783+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:45 smithi155 bash[27817]: cluster 2023-10-09T06:04:44.279159+0000 mgr.y (mgr.14430) 199 : cluster [DBG] pgmap v190: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:45 smithi155 bash[27817]: audit 2023-10-09T06:04:45.010533+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:45 smithi155 bash[27817]: audit 2023-10-09T06:04:45.015385+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:45 smithi155 bash[27817]: audit 2023-10-09T06:04:45.022783+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:04:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[27513]: cephadm 2023-10-09T06:04:45.013434+0000 mgr.y (mgr.14430) 200 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T06:04:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[27513]: cephadm 2023-10-09T06:04:45.013593+0000 mgr.y (mgr.14430) 201 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T06:04:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[27513]: cephadm 2023-10-09T06:04:45.026747+0000 mgr.y (mgr.14430) 202 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T06:04:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[27513]: cephadm 2023-10-09T06:04:45.252252+0000 mgr.y (mgr.14430) 203 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T06:04:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[33919]: cephadm 2023-10-09T06:04:45.013434+0000 mgr.y (mgr.14430) 200 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T06:04:46.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[33919]: cephadm 2023-10-09T06:04:45.013593+0000 mgr.y (mgr.14430) 201 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T06:04:46.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[33919]: cephadm 2023-10-09T06:04:45.026747+0000 mgr.y (mgr.14430) 202 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T06:04:46.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:46 smithi125 bash[33919]: cephadm 2023-10-09T06:04:45.252252+0000 mgr.y (mgr.14430) 203 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T06:04:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:46 smithi155 bash[27817]: cephadm 2023-10-09T06:04:45.013434+0000 mgr.y (mgr.14430) 200 : cephadm [INF] Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T06:04:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:46 smithi155 bash[27817]: cephadm 2023-10-09T06:04:45.013593+0000 mgr.y (mgr.14430) 201 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T06:04:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:46 smithi155 bash[27817]: cephadm 2023-10-09T06:04:45.026747+0000 mgr.y (mgr.14430) 202 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T06:04:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:46 smithi155 bash[27817]: cephadm 2023-10-09T06:04:45.252252+0000 mgr.y (mgr.14430) 203 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi155 2023-10-09T06:04:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:47 smithi125 bash[27513]: cluster 2023-10-09T06:04:46.280216+0000 mgr.y (mgr.14430) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:47 smithi125 bash[27513]: audit 2023-10-09T06:04:46.833493+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.125:0/2862759708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:47 smithi125 bash[33919]: cluster 2023-10-09T06:04:46.280216+0000 mgr.y (mgr.14430) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:47 smithi125 bash[33919]: audit 2023-10-09T06:04:46.833493+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.125:0/2862759708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:47 smithi155 bash[27817]: cluster 2023-10-09T06:04:46.280216+0000 mgr.y (mgr.14430) 204 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:47 smithi155 bash[27817]: audit 2023-10-09T06:04:46.833493+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.125:0/2862759708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:49.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:48] "GET /metrics HTTP/1.1" 200 192235 "" "Prometheus/2.33.4" 2023-10-09T06:04:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:49 smithi155 bash[27817]: cluster 2023-10-09T06:04:48.280843+0000 mgr.y (mgr.14430) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:49 smithi155 bash[27817]: audit 2023-10-09T06:04:49.245673+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.125:0/2510137932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:50.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:49 smithi125 bash[27513]: cluster 2023-10-09T06:04:48.280843+0000 mgr.y (mgr.14430) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:50.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:49 smithi125 bash[27513]: audit 2023-10-09T06:04:49.245673+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.125:0/2510137932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:49 smithi125 bash[33919]: cluster 2023-10-09T06:04:48.280843+0000 mgr.y (mgr.14430) 205 : cluster [DBG] pgmap v192: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:49 smithi125 bash[33919]: audit 2023-10-09T06:04:49.245673+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.125:0/2510137932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:51 smithi155 bash[27817]: cluster 2023-10-09T06:04:50.281900+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:51.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:04:51 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:04:52.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:51 smithi125 bash[27513]: cluster 2023-10-09T06:04:50.281900+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:52.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:51 smithi125 bash[33919]: cluster 2023-10-09T06:04:50.281900+0000 mgr.y (mgr.14430) 206 : cluster [DBG] pgmap v193: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:53.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:52 smithi125 bash[27513]: audit 2023-10-09T06:04:51.648021+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.125:0/3810829379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:53.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:52 smithi125 bash[33919]: audit 2023-10-09T06:04:51.648021+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.125:0/3810829379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:52 smithi155 bash[27817]: audit 2023-10-09T06:04:51.648021+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.125:0/3810829379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:54.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:53 smithi125 bash[27513]: cluster 2023-10-09T06:04:52.282558+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:54.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:53 smithi125 bash[33919]: cluster 2023-10-09T06:04:52.282558+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:53 smithi155 bash[27817]: cluster 2023-10-09T06:04:52.282558+0000 mgr.y (mgr.14430) 207 : cluster [DBG] pgmap v194: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:55.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:54 smithi125 bash[27513]: audit 2023-10-09T06:04:54.053485+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.125:0/2299090929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:55.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:54 smithi125 bash[33919]: audit 2023-10-09T06:04:54.053485+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.125:0/2299090929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:55.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:54 smithi155 bash[27817]: audit 2023-10-09T06:04:54.053485+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.125:0/2299090929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:56.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:55 smithi125 bash[27513]: cluster 2023-10-09T06:04:54.283245+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:56.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:55 smithi125 bash[33919]: cluster 2023-10-09T06:04:54.283245+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:55 smithi155 bash[27817]: cluster 2023-10-09T06:04:54.283245+0000 mgr.y (mgr.14430) 208 : cluster [DBG] pgmap v195: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:57.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:56 smithi125 bash[27513]: cluster 2023-10-09T06:04:56.284257+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:56 smithi125 bash[27513]: audit 2023-10-09T06:04:56.454777+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.125:0/2167964690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:57.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:56 smithi125 bash[33919]: cluster 2023-10-09T06:04:56.284257+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:57.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:56 smithi125 bash[33919]: audit 2023-10-09T06:04:56.454777+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.125:0/2167964690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:57.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:56 smithi155 bash[27817]: cluster 2023-10-09T06:04:56.284257+0000 mgr.y (mgr.14430) 209 : cluster [DBG] pgmap v196: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:57.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:56 smithi155 bash[27817]: audit 2023-10-09T06:04:56.454777+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.125:0/2167964690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:59.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:04:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:04:58] "GET /metrics HTTP/1.1" 200 192236 "" "Prometheus/2.33.4" 2023-10-09T06:04:59.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:59 smithi125 bash[33919]: cluster 2023-10-09T06:04:58.284889+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:59.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:04:59 smithi125 bash[33919]: audit 2023-10-09T06:04:58.870335+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.125:0/1035192735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:59 smithi125 bash[27513]: cluster 2023-10-09T06:04:58.284889+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:04:59 smithi125 bash[27513]: audit 2023-10-09T06:04:58.870335+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.125:0/1035192735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:59 smithi155 bash[27817]: cluster 2023-10-09T06:04:58.284889+0000 mgr.y (mgr.14430) 210 : cluster [DBG] pgmap v197: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:04:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:04:59 smithi155 bash[27817]: audit 2023-10-09T06:04:58.870335+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.125:0/1035192735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:01.827 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:01 smithi155 bash[27817]: cluster 2023-10-09T06:05:00.286039+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:01.827 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:01 smithi155 bash[27817]: audit 2023-10-09T06:05:01.278386+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.125:0/430564216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:01.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:01 smithi125 bash[27513]: cluster 2023-10-09T06:05:00.286039+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:01.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:01 smithi125 bash[27513]: audit 2023-10-09T06:05:01.278386+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.125:0/430564216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:01 smithi125 bash[33919]: cluster 2023-10-09T06:05:00.286039+0000 mgr.y (mgr.14430) 211 : cluster [DBG] pgmap v198: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:01 smithi125 bash[33919]: audit 2023-10-09T06:05:01.278386+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.125:0/430564216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:01 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:05:03.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:03 smithi125 bash[27513]: cluster 2023-10-09T06:05:02.286818+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:03.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:03 smithi125 bash[33919]: cluster 2023-10-09T06:05:02.286818+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:03 smithi155 bash[27817]: cluster 2023-10-09T06:05:02.286818+0000 mgr.y (mgr.14430) 212 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:04 smithi125 bash[27513]: audit 2023-10-09T06:05:03.699305+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.125:0/3134742378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:04 smithi125 bash[33919]: audit 2023-10-09T06:05:03.699305+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.125:0/3134742378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:04 smithi155 bash[27817]: audit 2023-10-09T06:05:03.699305+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.125:0/3134742378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:05.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:05 smithi125 bash[27513]: cluster 2023-10-09T06:05:04.287480+0000 mgr.y (mgr.14430) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:05 smithi125 bash[33919]: cluster 2023-10-09T06:05:04.287480+0000 mgr.y (mgr.14430) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:05 smithi155 bash[27817]: cluster 2023-10-09T06:05:04.287480+0000 mgr.y (mgr.14430) 213 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:06.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:06 smithi125 bash[27513]: audit 2023-10-09T06:05:06.107786+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.125:0/942616883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:06.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:06 smithi125 bash[33919]: audit 2023-10-09T06:05:06.107786+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.125:0/942616883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:06 smithi155 bash[27817]: audit 2023-10-09T06:05:06.107786+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.125:0/942616883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:07 smithi125 bash[33919]: cluster 2023-10-09T06:05:06.288598+0000 mgr.y (mgr.14430) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:07 smithi125 bash[27513]: cluster 2023-10-09T06:05:06.288598+0000 mgr.y (mgr.14430) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:07 smithi155 bash[27817]: cluster 2023-10-09T06:05:06.288598+0000 mgr.y (mgr.14430) 214 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:09.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:08] "GET /metrics HTTP/1.1" 200 192236 "" "Prometheus/2.33.4" 2023-10-09T06:05:09.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:09 smithi125 bash[27513]: cluster 2023-10-09T06:05:08.289213+0000 mgr.y (mgr.14430) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:09.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:09 smithi125 bash[27513]: audit 2023-10-09T06:05:08.517012+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.125:0/2815702189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:09 smithi125 bash[33919]: cluster 2023-10-09T06:05:08.289213+0000 mgr.y (mgr.14430) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:09 smithi125 bash[33919]: audit 2023-10-09T06:05:08.517012+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.125:0/2815702189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:09 smithi155 bash[27817]: cluster 2023-10-09T06:05:08.289213+0000 mgr.y (mgr.14430) 215 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:09 smithi155 bash[27817]: audit 2023-10-09T06:05:08.517012+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.125:0/2815702189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:11 smithi155 bash[27817]: cluster 2023-10-09T06:05:10.290219+0000 mgr.y (mgr.14430) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:11 smithi155 bash[27817]: audit 2023-10-09T06:05:10.928405+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.125:0/546795540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:11.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:11 smithi125 bash[27513]: cluster 2023-10-09T06:05:10.290219+0000 mgr.y (mgr.14430) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:11.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:11 smithi125 bash[27513]: audit 2023-10-09T06:05:10.928405+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.125:0/546795540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:11 smithi125 bash[33919]: cluster 2023-10-09T06:05:10.290219+0000 mgr.y (mgr.14430) 216 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:11 smithi125 bash[33919]: audit 2023-10-09T06:05:10.928405+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.125:0/546795540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:11 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:05:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:13 smithi125 bash[27513]: cluster 2023-10-09T06:05:12.290852+0000 mgr.y (mgr.14430) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:13 smithi125 bash[27513]: audit 2023-10-09T06:05:13.340192+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.125:0/1854511178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:13 smithi125 bash[33919]: cluster 2023-10-09T06:05:12.290852+0000 mgr.y (mgr.14430) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:13 smithi125 bash[33919]: audit 2023-10-09T06:05:13.340192+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.125:0/1854511178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:13 smithi155 bash[27817]: cluster 2023-10-09T06:05:12.290852+0000 mgr.y (mgr.14430) 217 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:13 smithi155 bash[27817]: audit 2023-10-09T06:05:13.340192+0000 mon.b (mon.1) 23 : audit [DBG] from='client.? 172.21.15.125:0/1854511178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:15.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:15 smithi125 bash[27513]: cluster 2023-10-09T06:05:14.291474+0000 mgr.y (mgr.14430) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:15 smithi125 bash[33919]: cluster 2023-10-09T06:05:14.291474+0000 mgr.y (mgr.14430) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:15 smithi155 bash[27817]: cluster 2023-10-09T06:05:14.291474+0000 mgr.y (mgr.14430) 218 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:16.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:16 smithi125 bash[27513]: audit 2023-10-09T06:05:15.750283+0000 mon.a (mon.0) 725 : audit [DBG] from='client.? 172.21.15.125:0/3462428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:16.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:16 smithi125 bash[33919]: audit 2023-10-09T06:05:15.750283+0000 mon.a (mon.0) 725 : audit [DBG] from='client.? 172.21.15.125:0/3462428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:16 smithi155 bash[27817]: audit 2023-10-09T06:05:15.750283+0000 mon.a (mon.0) 725 : audit [DBG] from='client.? 172.21.15.125:0/3462428015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:17.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:17 smithi125 bash[27513]: cluster 2023-10-09T06:05:16.292613+0000 mgr.y (mgr.14430) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:17 smithi125 bash[33919]: cluster 2023-10-09T06:05:16.292613+0000 mgr.y (mgr.14430) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:17 smithi155 bash[27817]: cluster 2023-10-09T06:05:16.292613+0000 mgr.y (mgr.14430) 219 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:18.749 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:18 smithi125 bash[27513]: audit 2023-10-09T06:05:18.159318+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.125:0/866446012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:18.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:18 smithi125 bash[33919]: audit 2023-10-09T06:05:18.159318+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.125:0/866446012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:18 smithi155 bash[27817]: audit 2023-10-09T06:05:18.159318+0000 mon.c (mon.2) 118 : audit [DBG] from='client.? 172.21.15.125:0/866446012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:19.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:18] "GET /metrics HTTP/1.1" 200 192236 "" "Prometheus/2.33.4" 2023-10-09T06:05:19.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:19 smithi125 bash[27513]: cluster 2023-10-09T06:05:18.293266+0000 mgr.y (mgr.14430) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:19 smithi125 bash[33919]: cluster 2023-10-09T06:05:18.293266+0000 mgr.y (mgr.14430) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:19 smithi155 bash[27817]: cluster 2023-10-09T06:05:18.293266+0000 mgr.y (mgr.14430) 220 : cluster [DBG] pgmap v207: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:21.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:21 smithi155 bash[27817]: cluster 2023-10-09T06:05:20.294126+0000 mgr.y (mgr.14430) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:21.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:21 smithi155 bash[27817]: audit 2023-10-09T06:05:20.569877+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.125:0/1507652879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:21.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:21 smithi125 bash[27513]: cluster 2023-10-09T06:05:20.294126+0000 mgr.y (mgr.14430) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:21.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:21 smithi125 bash[27513]: audit 2023-10-09T06:05:20.569877+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.125:0/1507652879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:21 smithi125 bash[33919]: cluster 2023-10-09T06:05:20.294126+0000 mgr.y (mgr.14430) 221 : cluster [DBG] pgmap v208: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:21 smithi125 bash[33919]: audit 2023-10-09T06:05:20.569877+0000 mon.a (mon.0) 726 : audit [DBG] from='client.? 172.21.15.125:0/1507652879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:21 smithi155 bash[28688]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:05:23.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:23 smithi125 bash[27513]: cluster 2023-10-09T06:05:22.294732+0000 mgr.y (mgr.14430) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:23.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:23 smithi125 bash[27513]: audit 2023-10-09T06:05:22.974382+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.125:0/2243529367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:23 smithi125 bash[33919]: cluster 2023-10-09T06:05:22.294732+0000 mgr.y (mgr.14430) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:23 smithi125 bash[33919]: audit 2023-10-09T06:05:22.974382+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.125:0/2243529367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:23 smithi155 bash[27817]: cluster 2023-10-09T06:05:22.294732+0000 mgr.y (mgr.14430) 222 : cluster [DBG] pgmap v209: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:23 smithi155 bash[27817]: audit 2023-10-09T06:05:22.974382+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.125:0/2243529367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: cluster 2023-10-09T06:05:24.295329+0000 mgr.y (mgr.14430) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.118453+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.121205+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.126260+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.127543+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.128579+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:25 smithi155 bash[27817]: audit 2023-10-09T06:05:25.380516+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.125:0/1977243218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: cluster 2023-10-09T06:05:24.295329+0000 mgr.y (mgr.14430) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.118453+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.121205+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.126260+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.127543+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:05:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.128579+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[33919]: audit 2023-10-09T06:05:25.380516+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.125:0/1977243218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:25.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: cluster 2023-10-09T06:05:24.295329+0000 mgr.y (mgr.14430) 223 : cluster [DBG] pgmap v210: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:25.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.118453+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:25.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.121205+0000 mon.c (mon.2) 119 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.126260+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:05:25.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.127543+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:05:25.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.128579+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:25.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:25 smithi125 bash[27513]: audit 2023-10-09T06:05:25.380516+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.125:0/1977243218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:26 smithi125 bash[27513]: cephadm 2023-10-09T06:05:25.112050+0000 mgr.y (mgr.14430) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:05:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:26 smithi125 bash[27513]: cephadm 2023-10-09T06:05:25.129518+0000 mgr.y (mgr.14430) 225 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:05:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:26 smithi125 bash[33919]: cephadm 2023-10-09T06:05:25.112050+0000 mgr.y (mgr.14430) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:05:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:26 smithi125 bash[33919]: cephadm 2023-10-09T06:05:25.129518+0000 mgr.y (mgr.14430) 225 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:05:26.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:26 smithi155 bash[27817]: cephadm 2023-10-09T06:05:25.112050+0000 mgr.y (mgr.14430) 224 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:05:26.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:26 smithi155 bash[27817]: cephadm 2023-10-09T06:05:25.129518+0000 mgr.y (mgr.14430) 225 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:05:27.252 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:27 smithi155 systemd[1]: Stopping Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:05:27.518 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:27 smithi155 bash[50411]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr.x 2023-10-09T06:05:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:27 smithi125 bash[27513]: cluster 2023-10-09T06:05:26.296506+0000 mgr.y (mgr.14430) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:27 smithi125 bash[33919]: cluster 2023-10-09T06:05:26.296506+0000 mgr.y (mgr.14430) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:27 smithi155 bash[27817]: cluster 2023-10-09T06:05:26.296506+0000 mgr.y (mgr.14430) 226 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:28.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:27 smithi155 bash[50449]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T06:05:28.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:28 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:05:28.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:28 smithi155 bash[50513]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T06:05:28.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:28 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T06:05:28.394 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:28 smithi155 systemd[1]: Stopped Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:05:28.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:28] "GET /metrics HTTP/1.1" 200 192237 "" "Prometheus/2.33.4" 2023-10-09T06:05:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[27513]: audit 2023-10-09T06:05:27.790028+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.125:0/1815895886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[27513]: audit 2023-10-09T06:05:28.504769+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[27513]: audit 2023-10-09T06:05:28.506665+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:28.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[33919]: audit 2023-10-09T06:05:27.790028+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.125:0/1815895886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:28.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[33919]: audit 2023-10-09T06:05:28.504769+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:28.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:28 smithi125 bash[33919]: audit 2023-10-09T06:05:28.506665+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:28.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:28 smithi155 systemd[1]: Started Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:05:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:28 smithi155 bash[27817]: audit 2023-10-09T06:05:27.790028+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.125:0/1815895886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:28.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:28 smithi155 bash[27817]: audit 2023-10-09T06:05:28.504769+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:28.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:28 smithi155 bash[27817]: audit 2023-10-09T06:05:28.506665+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:29.769 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:29 smithi155 bash[50569]: debug 2023-10-09T06:05:29.601+0000 7eff2c409700 1 -- 172.21.15.155:0/638742934 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560205c1f1e0 con 0x56020699e800 2023-10-09T06:05:29.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:29 smithi155 bash[27817]: cluster 2023-10-09T06:05:28.297173+0000 mgr.y (mgr.14430) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:29.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:29 smithi125 bash[27513]: cluster 2023-10-09T06:05:28.297173+0000 mgr.y (mgr.14430) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:29 smithi125 bash[33919]: cluster 2023-10-09T06:05:28.297173+0000 mgr.y (mgr.14430) 227 : cluster [DBG] pgmap v212: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:30.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:29 smithi155 bash[50569]: debug 2023-10-09T06:05:29.765+0000 7eff36557200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:05:30.799 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:30 smithi155 bash[27817]: audit 2023-10-09T06:05:30.208701+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.125:0/4151937911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:30.799 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:30 smithi155 bash[50569]: debug 2023-10-09T06:05:30.689+0000 7eff36557200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:05:30.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:30 smithi125 bash[27513]: audit 2023-10-09T06:05:30.208701+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.125:0/4151937911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:30.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:30 smithi125 bash[33919]: audit 2023-10-09T06:05:30.208701+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.125:0/4151937911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:31.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:30 smithi155 bash[50569]: debug 2023-10-09T06:05:30.797+0000 7eff36557200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:05:31.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:30 smithi155 bash[50569]: debug 2023-10-09T06:05:30.897+0000 7eff36557200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:05:31.530 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:31 smithi155 bash[50569]: debug 2023-10-09T06:05:31.161+0000 7eff36557200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:05:31.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:31 smithi125 bash[27513]: cluster 2023-10-09T06:05:30.298218+0000 mgr.y (mgr.14430) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:31.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:31 smithi125 bash[33919]: cluster 2023-10-09T06:05:30.298218+0000 mgr.y (mgr.14430) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:31 smithi155 bash[27817]: cluster 2023-10-09T06:05:30.298218+0000 mgr.y (mgr.14430) 228 : cluster [DBG] pgmap v213: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:31.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:31 smithi155 bash[50569]: debug 2023-10-09T06:05:31.525+0000 7eff36557200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:05:31.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:31 smithi155 bash[50569]: debug 2023-10-09T06:05:31.689+0000 7eff36557200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:05:32.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:31 smithi155 bash[50569]: debug 2023-10-09T06:05:31.905+0000 7eff36557200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:05:33.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:32 smithi155 bash[50569]: debug 2023-10-09T06:05:32.929+0000 7eff36557200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:05:33.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[50569]: debug 2023-10-09T06:05:33.045+0000 7eff36557200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:05:33.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[50569]: debug 2023-10-09T06:05:33.165+0000 7eff36557200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:05:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:33 smithi125 bash[27513]: cluster 2023-10-09T06:05:32.298703+0000 mgr.y (mgr.14430) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:33 smithi125 bash[27513]: audit 2023-10-09T06:05:32.616107+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.125:0/4050100724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:33.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:05:33 smithi125 bash[56935]: level=warn ts=2023-10-09T06:05:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi155.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.155:8443: connect: connection refused" 2023-10-09T06:05:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:33 smithi125 bash[33919]: cluster 2023-10-09T06:05:32.298703+0000 mgr.y (mgr.14430) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:33 smithi125 bash[33919]: audit 2023-10-09T06:05:32.616107+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.125:0/4050100724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[27817]: cluster 2023-10-09T06:05:32.298703+0000 mgr.y (mgr.14430) 229 : cluster [DBG] pgmap v214: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[27817]: audit 2023-10-09T06:05:32.616107+0000 mon.a (mon.0) 733 : audit [DBG] from='client.? 172.21.15.125:0/4050100724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:33.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[50569]: debug 2023-10-09T06:05:33.693+0000 7eff36557200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:05:34.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:33 smithi155 bash[50569]: debug 2023-10-09T06:05:33.941+0000 7eff36557200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:05:34.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:34 smithi155 bash[50569]: debug 2023-10-09T06:05:34.453+0000 7eff36557200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:05:35.230 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:34 smithi125 bash[27779]: ignoring --setuser ceph since I am not root 2023-10-09T06:05:35.231 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:34 smithi125 bash[27779]: ignoring --setgroup ceph since I am not root 2023-10-09T06:05:35.231 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:34 smithi125 bash[27779]: debug 2023-10-09T06:05:34.961+0000 7f69fb78c700 1 -- 172.21.15.125:0/618152643 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557a4d7784e0 con 0x557a4d86cc00 2023-10-09T06:05:35.231 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27779]: debug 2023-10-09T06:05:35.121+0000 7f6a0b954680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:05:35.257 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[50569]: debug 2023-10-09T06:05:35.025+0000 7eff36557200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:05:35.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27779]: debug 2023-10-09T06:05:35.225+0000 7f6a0b954680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.249666+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: cluster 2023-10-09T06:05:34.299253+0000 mgr.y (mgr.14430) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.580085+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.879988+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.881885+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.890081+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.898483+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.903995+0000 mon.c (mon.2) 127 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.904493+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: cluster 2023-10-09T06:05:34.913971+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:34.914391+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: cluster 2023-10-09T06:05:34.914464+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00933162s) 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[33919]: audit 2023-10-09T06:05:35.040196+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.125:0/721712927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.249666+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: cluster 2023-10-09T06:05:34.299253+0000 mgr.y (mgr.14430) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.580085+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.879988+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.881885+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.890081+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.898483+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.903995+0000 mon.c (mon.2) 127 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.904493+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: cluster 2023-10-09T06:05:34.913971+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:34.914391+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: cluster 2023-10-09T06:05:34.914464+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00933162s) 2023-10-09T06:05:35.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27513]: audit 2023-10-09T06:05:35.040196+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.125:0/721712927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.249666+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: cluster 2023-10-09T06:05:34.299253+0000 mgr.y (mgr.14430) 230 : cluster [DBG] pgmap v215: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:05:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.580085+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.879988+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.881885+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.890081+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14430 ' entity='mgr.y' 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.898483+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.903995+0000 mon.c (mon.2) 127 : audit [INF] from='mgr.14430 172.21.15.125:0/3029963033' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.904493+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: cluster 2023-10-09T06:05:34.913971+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:34.914391+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14430 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: cluster 2023-10-09T06:05:34.914464+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00933162s) 2023-10-09T06:05:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[27817]: audit 2023-10-09T06:05:35.040196+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.125:0/721712927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:36.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:35 smithi125 bash[27779]: debug 2023-10-09T06:05:35.709+0000 7f6a0b954680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:05:36.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[50569]: debug 2023-10-09T06:05:35.645+0000 7eff36557200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:05:36.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:35 smithi155 bash[50569]: debug 2023-10-09T06:05:35.769+0000 7eff36557200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:05:36.442 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.157+0000 7eff36557200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:05:36.442 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.281+0000 7eff36557200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:05:36.708 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.437+0000 7eff36557200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:05:36.708 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.565+0000 7eff36557200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:05:37.022 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:36 smithi125 bash[27779]: debug 2023-10-09T06:05:36.665+0000 7f6a0b954680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:05:37.022 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:36 smithi125 bash[27779]: debug 2023-10-09T06:05:36.765+0000 7f6a0b954680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:05:37.050 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.705+0000 7eff36557200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:05:37.050 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:36 smithi155 bash[50569]: debug 2023-10-09T06:05:36.909+0000 7eff36557200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:05:37.385 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:37 smithi125 bash[27779]: debug 2023-10-09T06:05:37.017+0000 7f6a0b954680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:05:37.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:37 smithi155 bash[50569]: debug 2023-10-09T06:05:37.045+0000 7eff36557200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:05:37.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:37 smithi125 bash[27513]: audit 2023-10-09T06:05:37.441772+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.125:0/330988356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:37 smithi125 bash[33919]: audit 2023-10-09T06:05:37.441772+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.125:0/330988356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:37 smithi155 bash[27817]: audit 2023-10-09T06:05:37.441772+0000 mon.a (mon.0) 741 : audit [DBG] from='client.? 172.21.15.125:0/330988356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:38.447 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27779]: debug 2023-10-09T06:05:38.177+0000 7f6a0b954680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:05:38.448 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27779]: debug 2023-10-09T06:05:38.345+0000 7f6a0b954680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:05:38.593 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.281+0000 7eff36557200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:05:38.593 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STARTING 2023-10-09T06:05:38.593 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: CherryPy Checker: 2023-10-09T06:05:38.593 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: The Application mounted at '' has an empty config. 2023-10-09T06:05:38.593 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Serving on http://:::9283 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STARTED 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STOPPING 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STOPPED 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STARTING 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: cluster 2023-10-09T06:05:38.304671+0000 mon.a (mon.0) 742 : cluster [INF] Active manager daemon x restarted 2023-10-09T06:05:38.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: cluster 2023-10-09T06:05:38.305307+0000 mon.a (mon.0) 743 : cluster [INF] Activating manager daemon x 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: cluster 2023-10-09T06:05:38.314162+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: cluster 2023-10-09T06:05:38.314485+0000 mon.a (mon.0) 745 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00943132s) 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.317961+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.318210+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.318495+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.318812+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.319195+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.319469+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.319782+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.320061+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:05:38.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.320300+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.320510+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.320728+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.320938+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.321161+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.321366+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.321593+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.322220+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.322545+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.323063+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.323281+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.324262+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:05:38.597 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[27817]: audit 2023-10-09T06:05:38.324676+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:38.731 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27779]: debug 2023-10-09T06:05:38.441+0000 7f6a0b954680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:05:38.731 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27779]: debug 2023-10-09T06:05:38.561+0000 7f6a0b954680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:05:38.731 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: cluster 2023-10-09T06:05:38.304671+0000 mon.a (mon.0) 742 : cluster [INF] Active manager daemon x restarted 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: cluster 2023-10-09T06:05:38.305307+0000 mon.a (mon.0) 743 : cluster [INF] Activating manager daemon x 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: cluster 2023-10-09T06:05:38.314162+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: cluster 2023-10-09T06:05:38.314485+0000 mon.a (mon.0) 745 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00943132s) 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.317961+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.318210+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.318495+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.318812+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.319195+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.319469+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.319782+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:05:38.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.320061+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.320300+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.320510+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.320728+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.320938+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.321161+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.321366+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.321593+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:05:38.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: cluster 2023-10-09T06:05:38.304671+0000 mon.a (mon.0) 742 : cluster [INF] Active manager daemon x restarted 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: cluster 2023-10-09T06:05:38.305307+0000 mon.a (mon.0) 743 : cluster [INF] Activating manager daemon x 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: cluster 2023-10-09T06:05:38.314162+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: cluster 2023-10-09T06:05:38.314485+0000 mon.a (mon.0) 745 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00943132s) 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.317961+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.318210+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.318495+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.318812+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.319195+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.319469+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.319782+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:05:38.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.320061+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.320300+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.320510+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.320728+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.320938+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.321161+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.321366+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.321593+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.322220+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:05:38.735 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.322545+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.323063+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.323281+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.324262+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27513]: audit 2023-10-09T06:05:38.324676+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.322220+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.322545+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.323063+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.323281+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.324262+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:05:38.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[33919]: audit 2023-10-09T06:05:38.324676+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:38.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe939f4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.894 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe939f4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.894 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe939f4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.894 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe939f4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.894 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe939f4700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.894 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe961f9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe961f9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe961f9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe961f9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: debug 2023-10-09T06:05:38.685+0000 7efe961f9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Serving on http://:::9283 2023-10-09T06:05:38.895 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:38 smithi155 bash[50569]: [09/Oct/2023:06:05:38] ENGINE Bus STARTED 2023-10-09T06:05:39.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:38 smithi125 bash[27779]: debug 2023-10-09T06:05:38.805+0000 7f6a0b954680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:05:39.501 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27779]: debug 2023-10-09T06:05:39.180+0000 7f6a0b954680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:05:39.501 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27779]: debug 2023-10-09T06:05:39.304+0000 7f6a0b954680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:05:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: cluster 2023-10-09T06:05:38.507105+0000 mon.a (mon.0) 746 : cluster [INF] Manager daemon x is now available 2023-10-09T06:05:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.538715+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.548530+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.550939+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.566829+0000 mon.b (mon.1) 47 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.566987+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.589445+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.598919+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.599009+0000 mon.b (mon.1) 49 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.633748+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.633771+0000 mon.b (mon.1) 50 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:38.638067+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:39.284449+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[27513]: audit 2023-10-09T06:05:39.298324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: cluster 2023-10-09T06:05:38.507105+0000 mon.a (mon.0) 746 : cluster [INF] Manager daemon x is now available 2023-10-09T06:05:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.538715+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.548530+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.550939+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.566829+0000 mon.b (mon.1) 47 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.566987+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.589445+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.598919+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.599009+0000 mon.b (mon.1) 49 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.633748+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.633771+0000 mon.b (mon.1) 50 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:38.638067+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:39.284449+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:39 smithi125 bash[33919]: audit 2023-10-09T06:05:39.298324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: cluster 2023-10-09T06:05:38.507105+0000 mon.a (mon.0) 746 : cluster [INF] Manager daemon x is now available 2023-10-09T06:05:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.538715+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.548530+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.550939+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.566829+0000 mon.b (mon.1) 47 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.566987+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.589445+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.598919+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.599009+0000 mon.b (mon.1) 49 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.633748+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.633771+0000 mon.b (mon.1) 50 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:38.638067+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:39.284449+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:39 smithi155 bash[27817]: audit 2023-10-09T06:05:39.298324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:40.513 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[56935]: level=info ts=2023-10-09T06:05:40.201Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2023-10-09T06:05:40.513 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27779]: debug 2023-10-09T06:05:40.196+0000 7f6a0b954680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:05:40.514 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27779]: debug 2023-10-09T06:05:40.304+0000 7f6a0b954680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:05:40.514 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27779]: debug 2023-10-09T06:05:40.424+0000 7f6a0b954680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:05:40.885 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27779]: debug 2023-10-09T06:05:40.664+0000 7f6a0b954680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: cephadm 2023-10-09T06:05:39.071248+0000 mgr.x (mgr.25270) 1 : cephadm [INF] Deploying cephadm binary to smithi155 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: cephadm 2023-10-09T06:05:39.077299+0000 mgr.x (mgr.25270) 2 : cephadm [INF] Deploying cephadm binary to smithi125 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: cluster 2023-10-09T06:05:39.514555+0000 mon.a (mon.0) 754 : cluster [DBG] mgrmap e21: x(active, since 1.2095s) 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: audit 2023-10-09T06:05:39.541296+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: cluster 2023-10-09T06:05:39.541508+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:05:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: cluster 2023-10-09T06:05:39.556566+0000 mgr.x (mgr.25270) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 GiB/s rd, 1.8 GiB/s wr, 1.37M op/s 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: audit 2023-10-09T06:05:39.559845+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: audit 2023-10-09T06:05:39.560048+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[33919]: audit 2023-10-09T06:05:39.880311+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.125:0/2846195518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: cephadm 2023-10-09T06:05:39.071248+0000 mgr.x (mgr.25270) 1 : cephadm [INF] Deploying cephadm binary to smithi155 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: cephadm 2023-10-09T06:05:39.077299+0000 mgr.x (mgr.25270) 2 : cephadm [INF] Deploying cephadm binary to smithi125 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: cluster 2023-10-09T06:05:39.514555+0000 mon.a (mon.0) 754 : cluster [DBG] mgrmap e21: x(active, since 1.2095s) 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: audit 2023-10-09T06:05:39.541296+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: cluster 2023-10-09T06:05:39.541508+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: cluster 2023-10-09T06:05:39.556566+0000 mgr.x (mgr.25270) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 GiB/s rd, 1.8 GiB/s wr, 1.37M op/s 2023-10-09T06:05:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: audit 2023-10-09T06:05:39.559845+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: audit 2023-10-09T06:05:39.560048+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27513]: audit 2023-10-09T06:05:39.880311+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.125:0/2846195518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: cephadm 2023-10-09T06:05:39.071248+0000 mgr.x (mgr.25270) 1 : cephadm [INF] Deploying cephadm binary to smithi155 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: cephadm 2023-10-09T06:05:39.077299+0000 mgr.x (mgr.25270) 2 : cephadm [INF] Deploying cephadm binary to smithi125 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: cluster 2023-10-09T06:05:39.514555+0000 mon.a (mon.0) 754 : cluster [DBG] mgrmap e21: x(active, since 1.2095s) 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: audit 2023-10-09T06:05:39.541296+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: cluster 2023-10-09T06:05:39.541508+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:05:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: cluster 2023-10-09T06:05:39.556566+0000 mgr.x (mgr.25270) 3 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 GiB/s rd, 1.8 GiB/s wr, 1.37M op/s 2023-10-09T06:05:40.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: audit 2023-10-09T06:05:39.559845+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: audit 2023-10-09T06:05:39.560048+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:05:40.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:40 smithi155 bash[27817]: audit 2023-10-09T06:05:39.880311+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.125:0/2846195518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:41.357 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:40 smithi125 bash[27779]: debug 2023-10-09T06:05:40.988+0000 7f6a0b954680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:05:41.358 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: debug 2023-10-09T06:05:41.164+0000 7f6a0b954680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:05:41.635 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: debug 2023-10-09T06:05:41.352+0000 7f6a0b954680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: debug 2023-10-09T06:05:41.596+0000 7f6a0b954680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: cephadm 2023-10-09T06:05:40.304454+0000 mgr.x (mgr.25270) 4 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTING 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: cluster 2023-10-09T06:05:40.322408+0000 mgr.x (mgr.25270) 5 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 11 MiB/s wr, 8.10k op/s 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: cephadm 2023-10-09T06:05:40.517359+0000 mgr.x (mgr.25270) 6 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: audit 2023-10-09T06:05:40.553341+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:41.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: cluster 2023-10-09T06:05:40.553442+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27513]: cluster 2023-10-09T06:05:40.554000+0000 mon.a (mon.0) 760 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: cephadm 2023-10-09T06:05:40.304454+0000 mgr.x (mgr.25270) 4 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTING 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: cluster 2023-10-09T06:05:40.322408+0000 mgr.x (mgr.25270) 5 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 11 MiB/s wr, 8.10k op/s 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: cephadm 2023-10-09T06:05:40.517359+0000 mgr.x (mgr.25270) 6 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: audit 2023-10-09T06:05:40.553341+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: cluster 2023-10-09T06:05:40.553442+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:05:41.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[33919]: cluster 2023-10-09T06:05:40.554000+0000 mon.a (mon.0) 760 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T06:05:41.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: cephadm 2023-10-09T06:05:40.304454+0000 mgr.x (mgr.25270) 4 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTING 2023-10-09T06:05:41.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: cluster 2023-10-09T06:05:40.322408+0000 mgr.x (mgr.25270) 5 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 11 MiB/s wr, 8.10k op/s 2023-10-09T06:05:41.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: cephadm 2023-10-09T06:05:40.517359+0000 mgr.x (mgr.25270) 6 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on https://172.21.15.155:7150 2023-10-09T06:05:41.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: audit 2023-10-09T06:05:40.553341+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:05:41.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: cluster 2023-10-09T06:05:40.553442+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:05:41.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[27817]: cluster 2023-10-09T06:05:40.554000+0000 mon.a (mon.0) 760 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-09T06:05:42.135 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: debug 2023-10-09T06:05:41.708+0000 7f6a0b954680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:05:42.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: [09/Oct/2023:06:05:41] ENGINE Bus STARTING 2023-10-09T06:05:42.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: CherryPy Checker: 2023-10-09T06:05:42.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: The Application mounted at '' has an empty config. 2023-10-09T06:05:42.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: [09/Oct/2023:06:05:41] ENGINE Serving on http://:::9283 2023-10-09T06:05:42.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:41 smithi125 bash[27779]: [09/Oct/2023:06:05:41] ENGINE Bus STARTED 2023-10-09T06:05:42.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:41 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:41] "GET /metrics HTTP/1.1" 200 31048 "" "Prometheus/2.33.4" 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: cephadm 2023-10-09T06:05:40.649737+0000 mgr.x (mgr.25270) 7 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: cephadm 2023-10-09T06:05:40.650523+0000 mgr.x (mgr.25270) 8 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTED 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:41.717162+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: cluster 2023-10-09T06:05:41.732929+0000 mon.a (mon.0) 762 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:41.735726+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:41.740481+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:41.744728+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:41.745300+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[27513]: audit 2023-10-09T06:05:42.393473+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.125:0/3018702642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: cephadm 2023-10-09T06:05:40.649737+0000 mgr.x (mgr.25270) 7 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: cephadm 2023-10-09T06:05:40.650523+0000 mgr.x (mgr.25270) 8 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTED 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:41.717162+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: cluster 2023-10-09T06:05:41.732929+0000 mon.a (mon.0) 762 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:41.735726+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:41.740481+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:41.744728+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:05:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:41.745300+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:42 smithi125 bash[33919]: audit 2023-10-09T06:05:42.393473+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.125:0/3018702642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: cephadm 2023-10-09T06:05:40.649737+0000 mgr.x (mgr.25270) 7 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Serving on http://172.21.15.155:8765 2023-10-09T06:05:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: cephadm 2023-10-09T06:05:40.650523+0000 mgr.x (mgr.25270) 8 : cephadm [INF] [09/Oct/2023:06:05:40] ENGINE Bus STARTED 2023-10-09T06:05:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:41.717162+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: cluster 2023-10-09T06:05:41.732929+0000 mon.a (mon.0) 762 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:05:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:41.735726+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:05:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:41.740481+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:05:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:41.744728+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:05:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:41.745300+0000 mon.a (mon.0) 766 : audit [DBG] from='mgr.? 172.21.15.125:0/2484331844' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:05:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:42 smithi155 bash[27817]: audit 2023-10-09T06:05:42.393473+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.125:0/3018702642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[27513]: cluster 2023-10-09T06:05:42.322941+0000 mgr.x (mgr.25270) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.0 MiB/s wr, 2.25k op/s 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[27513]: cluster 2023-10-09T06:05:42.729488+0000 mon.a (mon.0) 768 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[27513]: audit 2023-10-09T06:05:42.732966+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[33919]: cluster 2023-10-09T06:05:42.322941+0000 mgr.x (mgr.25270) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.0 MiB/s wr, 2.25k op/s 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[33919]: cluster 2023-10-09T06:05:42.729488+0000 mon.a (mon.0) 768 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T06:05:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:43 smithi125 bash[33919]: audit 2023-10-09T06:05:42.732966+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:05:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:43 smithi155 bash[27817]: cluster 2023-10-09T06:05:42.322941+0000 mgr.x (mgr.25270) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 3.0 MiB/s wr, 2.25k op/s 2023-10-09T06:05:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:43 smithi155 bash[27817]: cluster 2023-10-09T06:05:42.729488+0000 mon.a (mon.0) 768 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-09T06:05:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:43 smithi155 bash[27817]: audit 2023-10-09T06:05:42.732966+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[27513]: cluster 2023-10-09T06:05:44.323653+0000 mgr.x (mgr.25270) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 1.31k op/s 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[27513]: audit 2023-10-09T06:05:44.805015+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.125:0/3068508366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[27513]: audit 2023-10-09T06:05:45.406320+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[33919]: cluster 2023-10-09T06:05:44.323653+0000 mgr.x (mgr.25270) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 1.31k op/s 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[33919]: audit 2023-10-09T06:05:44.805015+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.125:0/3068508366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:45 smithi125 bash[33919]: audit 2023-10-09T06:05:45.406320+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:45 smithi155 bash[27817]: cluster 2023-10-09T06:05:44.323653+0000 mgr.x (mgr.25270) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 1.31k op/s 2023-10-09T06:05:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:45 smithi155 bash[27817]: audit 2023-10-09T06:05:44.805015+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.125:0/3068508366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:45 smithi155 bash[27817]: audit 2023-10-09T06:05:45.406320+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[27513]: cluster 2023-10-09T06:05:46.324552+0000 mgr.x (mgr.25270) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 953 op/s 2023-10-09T06:05:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[27513]: audit 2023-10-09T06:05:47.257936+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.125:0/3513603596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[27513]: audit 2023-10-09T06:05:47.517052+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[27513]: audit 2023-10-09T06:05:47.530557+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[33919]: cluster 2023-10-09T06:05:46.324552+0000 mgr.x (mgr.25270) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 953 op/s 2023-10-09T06:05:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[33919]: audit 2023-10-09T06:05:47.257936+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.125:0/3513603596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[33919]: audit 2023-10-09T06:05:47.517052+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:47 smithi125 bash[33919]: audit 2023-10-09T06:05:47.530557+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:47 smithi155 bash[27817]: cluster 2023-10-09T06:05:46.324552+0000 mgr.x (mgr.25270) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 953 op/s 2023-10-09T06:05:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:47 smithi155 bash[27817]: audit 2023-10-09T06:05:47.257936+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.125:0/3513603596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:47 smithi155 bash[27817]: audit 2023-10-09T06:05:47.517052+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:47 smithi155 bash[27817]: audit 2023-10-09T06:05:47.530557+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:49.121 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:48 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[27513]: audit 2023-10-09T06:05:47.883496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[27513]: audit 2023-10-09T06:05:47.893756+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[27513]: audit 2023-10-09T06:05:48.200399+0000 mon.a (mon.0) 775 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[33919]: audit 2023-10-09T06:05:47.883496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[33919]: audit 2023-10-09T06:05:47.893756+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:49.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:49 smithi125 bash[33919]: audit 2023-10-09T06:05:48.200399+0000 mon.a (mon.0) 775 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:47.883496+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:47.893756+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:48.200399+0000 mon.a (mon.0) 775 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:51.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:48.201625+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:48.201998+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:48.202174+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: cluster 2023-10-09T06:05:48.325145+0000 mgr.x (mgr.25270) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:49.115290+0000 mon.a (mon.0) 776 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:49.119919+0000 mon.b (mon.1) 56 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:49.674348+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.125:0/1125773955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.029445+0000 mon.b (mon.1) 57 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.033805+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.033973+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.034193+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.038049+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.043266+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.045553+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.045906+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[27513]: audit 2023-10-09T06:05:50.046375+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:48.201625+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:48.201998+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:48.202174+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: cluster 2023-10-09T06:05:48.325145+0000 mgr.x (mgr.25270) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:49.115290+0000 mon.a (mon.0) 776 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:49.119919+0000 mon.b (mon.1) 56 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:49.674348+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.125:0/1125773955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.029445+0000 mon.b (mon.1) 57 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.033805+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.033973+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.034193+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.038049+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.043266+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.045553+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.045906+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:50 smithi125 bash[33919]: audit 2023-10-09T06:05:50.046375+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:48.201625+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:48.201998+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:48.202174+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: cluster 2023-10-09T06:05:48.325145+0000 mgr.x (mgr.25270) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 186 KiB/s wr, 24 op/s 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:49.115290+0000 mon.a (mon.0) 776 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:49.119919+0000 mon.b (mon.1) 56 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:49.674348+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.125:0/1125773955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.029445+0000 mon.b (mon.1) 57 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.033805+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.033973+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.034193+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:51.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.038049+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.043266+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:51.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.045553+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:05:51.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.045906+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:05:51.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:50 smithi155 bash[27817]: audit 2023-10-09T06:05:50.046375+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:05:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:51] "GET /metrics HTTP/1.1" 200 31048 "" "Prometheus/2.33.4" 2023-10-09T06:05:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[27817]: audit 2023-10-09T06:05:50.040926+0000 mon.c (mon.2) 132 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[27817]: cluster 2023-10-09T06:05:50.326367+0000 mgr.x (mgr.25270) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T06:05:52.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[27817]: audit 2023-10-09T06:05:50.950738+0000 mon.c (mon.2) 133 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:52.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[27817]: audit 2023-10-09T06:05:50.962237+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:51 smithi155 bash[27817]: audit 2023-10-09T06:05:50.977208+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[27513]: audit 2023-10-09T06:05:50.040926+0000 mon.c (mon.2) 132 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[27513]: cluster 2023-10-09T06:05:50.326367+0000 mgr.x (mgr.25270) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T06:05:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[27513]: audit 2023-10-09T06:05:50.950738+0000 mon.c (mon.2) 133 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[27513]: audit 2023-10-09T06:05:50.962237+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[27513]: audit 2023-10-09T06:05:50.977208+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[33919]: audit 2023-10-09T06:05:50.040926+0000 mon.c (mon.2) 132 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T06:05:52.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[33919]: cluster 2023-10-09T06:05:50.326367+0000 mgr.x (mgr.25270) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 25 op/s 2023-10-09T06:05:52.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[33919]: audit 2023-10-09T06:05:50.950738+0000 mon.c (mon.2) 133 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T06:05:52.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[33919]: audit 2023-10-09T06:05:50.962237+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:51 smithi125 bash[33919]: audit 2023-10-09T06:05:50.977208+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:05:52.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:05:52 smithi155 bash[48405]: ts=2023-10-09T06:05:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.125:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:53.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:52 smithi125 bash[27513]: audit 2023-10-09T06:05:52.092584+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.125:0/629023999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:53.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:52 smithi125 bash[27513]: cluster 2023-10-09T06:05:52.327069+0000 mgr.x (mgr.25270) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T06:05:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:52 smithi125 bash[33919]: audit 2023-10-09T06:05:52.092584+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.125:0/629023999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:52 smithi125 bash[33919]: cluster 2023-10-09T06:05:52.327069+0000 mgr.x (mgr.25270) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T06:05:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:52 smithi155 bash[27817]: audit 2023-10-09T06:05:52.092584+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.125:0/629023999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:52 smithi155 bash[27817]: cluster 2023-10-09T06:05:52.327069+0000 mgr.x (mgr.25270) 14 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 183 KiB/s wr, 21 op/s 2023-10-09T06:05:54.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:53 smithi125 bash[27513]: audit 2023-10-09T06:05:53.590268+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:54.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:53 smithi125 bash[33919]: audit 2023-10-09T06:05:53.590268+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:53 smithi155 bash[27817]: audit 2023-10-09T06:05:53.590268+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:55.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:54 smithi125 bash[27513]: cluster 2023-10-09T06:05:54.327679+0000 mgr.x (mgr.25270) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T06:05:55.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:54 smithi125 bash[27513]: audit 2023-10-09T06:05:54.537258+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.125:0/2656307305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:54 smithi125 bash[33919]: cluster 2023-10-09T06:05:54.327679+0000 mgr.x (mgr.25270) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T06:05:55.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:54 smithi125 bash[33919]: audit 2023-10-09T06:05:54.537258+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.125:0/2656307305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:54 smithi155 bash[27817]: cluster 2023-10-09T06:05:54.327679+0000 mgr.x (mgr.25270) 15 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 20 op/s 2023-10-09T06:05:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:54 smithi155 bash[27817]: audit 2023-10-09T06:05:54.537258+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.125:0/2656307305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:57.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:57 smithi125 bash[27513]: cluster 2023-10-09T06:05:56.328355+0000 mgr.x (mgr.25270) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T06:05:57.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:57 smithi125 bash[27513]: audit 2023-10-09T06:05:56.978429+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.125:0/1148608875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:57.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:57 smithi125 bash[33919]: cluster 2023-10-09T06:05:56.328355+0000 mgr.x (mgr.25270) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T06:05:57.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:57 smithi125 bash[33919]: audit 2023-10-09T06:05:56.978429+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.125:0/1148608875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:57 smithi155 bash[27817]: cluster 2023-10-09T06:05:56.328355+0000 mgr.x (mgr.25270) 16 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-09T06:05:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:57 smithi155 bash[27817]: audit 2023-10-09T06:05:56.978429+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.125:0/1148608875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:59.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:05:58 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:05:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:05:59.740 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:59 smithi125 bash[33919]: cluster 2023-10-09T06:05:58.329039+0000 mgr.x (mgr.25270) 17 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:05:59.740 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:05:59 smithi125 bash[33919]: audit 2023-10-09T06:05:59.409565+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.125:0/1991347752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:59.740 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:59 smithi125 bash[27513]: cluster 2023-10-09T06:05:58.329039+0000 mgr.x (mgr.25270) 17 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:05:59.740 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:05:59 smithi125 bash[27513]: audit 2023-10-09T06:05:59.409565+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.125:0/1991347752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:05:59.812 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:59 smithi155 bash[27817]: cluster 2023-10-09T06:05:58.329039+0000 mgr.x (mgr.25270) 17 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:05:59.813 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:05:59 smithi155 bash[27817]: audit 2023-10-09T06:05:59.409565+0000 mon.a (mon.0) 784 : audit [DBG] from='client.? 172.21.15.125:0/1991347752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: cluster 2023-10-09T06:06:00.330130+0000 mgr.x (mgr.25270) 18 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:06:01.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.365437+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.378061+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.381808+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.381914+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.384328+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.384499+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.386648+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.386750+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.389048+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.389150+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.391313+0000 mgr.x (mgr.25270) 19 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 2524M 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.399071+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.631150+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.642157+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.645602+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.645806+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.648470+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.648484+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: cluster 2023-10-09T06:06:00.330130+0000 mgr.x (mgr.25270) 18 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:06:01.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.365437+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.378061+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.381808+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.381914+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.384328+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.384499+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.386648+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.386750+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.389048+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.389150+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.391313+0000 mgr.x (mgr.25270) 19 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 2524M 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.399071+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.631150+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.642157+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.645602+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.645806+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.648470+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.648484+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.650739+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.650957+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.652941+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.653111+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.663141+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.666790+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:00.668116+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:01.341839+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[27513]: audit 2023-10-09T06:06:01.353170+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.650739+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.650957+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.652941+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.653111+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.663141+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.666790+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:00.668116+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:01.341839+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[33919]: audit 2023-10-09T06:06:01.353170+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: cluster 2023-10-09T06:06:00.330130+0000 mgr.x (mgr.25270) 18 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.365437+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.378061+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.381808+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.381914+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.384328+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.384499+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.386648+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.386750+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.389048+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.389150+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.391313+0000 mgr.x (mgr.25270) 19 : cephadm [INF] Adjusting osd_memory_target on smithi125 to 2524M 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.399071+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.631150+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.642157+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.645602+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.645806+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.648470+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.648484+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.650739+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.650957+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.652941+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.653111+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.663141+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.666790+0000 mon.b (mon.1) 73 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:00.668116+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:01.341839+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:01.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[27817]: audit 2023-10-09T06:06:01.353170+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:01 smithi125 systemd[1]: Stopping Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:02.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:01 smithi125 bash[71946]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter.a 2023-10-09T06:06:02.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:01 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:01] "GET /metrics HTTP/1.1" 200 33931 "" "Prometheus/2.33.4" 2023-10-09T06:06:02.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.654954+0000 mgr.x (mgr.25270) 20 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 2524M 2023-10-09T06:06:02.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.669902+0000 mgr.x (mgr.25270) 21 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:06:02.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.670251+0000 mgr.x (mgr.25270) 22 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:06:02.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.834838+0000 mgr.x (mgr.25270) 23 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.836812+0000 mgr.x (mgr.25270) 24 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:00.999231+0000 mgr.x (mgr.25270) 25 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:01.001827+0000 mgr.x (mgr.25270) 26 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:01.163865+0000 mgr.x (mgr.25270) 27 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:01.168723+0000 mgr.x (mgr.25270) 28 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.389464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.402481+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.413124+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.492200+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.498465+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.504991+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.511066+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:01.514050+0000 mgr.x (mgr.25270) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: cephadm 2023-10-09T06:06:01.514886+0000 mgr.x (mgr.25270) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[33919]: audit 2023-10-09T06:06:01.855686+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.125:0/1533500325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.654954+0000 mgr.x (mgr.25270) 20 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 2524M 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.669902+0000 mgr.x (mgr.25270) 21 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.670251+0000 mgr.x (mgr.25270) 22 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.834838+0000 mgr.x (mgr.25270) 23 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.836812+0000 mgr.x (mgr.25270) 24 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:00.999231+0000 mgr.x (mgr.25270) 25 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:01.001827+0000 mgr.x (mgr.25270) 26 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:01.163865+0000 mgr.x (mgr.25270) 27 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:01.168723+0000 mgr.x (mgr.25270) 28 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.389464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.402481+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.413124+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.492200+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.498465+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.504991+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.511066+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:01.514050+0000 mgr.x (mgr.25270) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:06:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: cephadm 2023-10-09T06:06:01.514886+0000 mgr.x (mgr.25270) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T06:06:02.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[27513]: audit 2023-10-09T06:06:01.855686+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.125:0/1533500325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:02.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[48405]: ts=2023-10-09T06:06:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.125:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.654954+0000 mgr.x (mgr.25270) 20 : cephadm [INF] Adjusting osd_memory_target on smithi155 to 2524M 2023-10-09T06:06:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.669902+0000 mgr.x (mgr.25270) 21 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:06:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.670251+0000 mgr.x (mgr.25270) 22 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.834838+0000 mgr.x (mgr.25270) 23 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.836812+0000 mgr.x (mgr.25270) 24 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:00.999231+0000 mgr.x (mgr.25270) 25 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:01.001827+0000 mgr.x (mgr.25270) 26 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:01.163865+0000 mgr.x (mgr.25270) 27 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:01.168723+0000 mgr.x (mgr.25270) 28 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.389464+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.402481+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.413124+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.492200+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.498465+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.504991+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.511066+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:01.514050+0000 mgr.x (mgr.25270) 29 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: cephadm 2023-10-09T06:06:01.514886+0000 mgr.x (mgr.25270) 30 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi125 2023-10-09T06:06:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:02 smithi155 bash[27817]: audit 2023-10-09T06:06:01.855686+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.125:0/1533500325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:02.939 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[71985]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T06:06:02.939 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:06:02.939 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 bash[72049]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T06:06:02.939 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:06:02.939 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 systemd[1]: Stopped Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:02.940 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:02 smithi125 systemd[1]: Started Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:03.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[27513]: cluster 2023-10-09T06:06:02.330791+0000 mgr.x (mgr.25270) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:03.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[27513]: audit 2023-10-09T06:06:02.914951+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:03.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[27513]: audit 2023-10-09T06:06:02.924364+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:03.883 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[33919]: cluster 2023-10-09T06:06:02.330791+0000 mgr.x (mgr.25270) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:03.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[33919]: audit 2023-10-09T06:06:02.914951+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:03.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:03 smithi125 bash[33919]: audit 2023-10-09T06:06:02.924364+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:03 smithi155 bash[27817]: cluster 2023-10-09T06:06:02.330791+0000 mgr.x (mgr.25270) 31 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:03 smithi155 bash[27817]: audit 2023-10-09T06:06:02.914951+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:03 smithi155 bash[27817]: audit 2023-10-09T06:06:02.924364+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:04.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.024Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T06:06:04.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.024Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T06:06:04.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.024Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.024Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T06:06:04.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T06:06:04.138 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T06:06:04.139 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T06:06:04.140 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T06:06:04.141 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T06:06:04.142 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T06:06:04.143 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.025Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T06:06:04.144 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[72077]: ts=2023-10-09T06:06:04.026Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[33919]: cephadm 2023-10-09T06:06:02.933902+0000 mgr.x (mgr.25270) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[33919]: cephadm 2023-10-09T06:06:02.951868+0000 mgr.x (mgr.25270) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[33919]: audit 2023-10-09T06:06:04.290175+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.125:0/2378448016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[27513]: cephadm 2023-10-09T06:06:02.933902+0000 mgr.x (mgr.25270) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[27513]: cephadm 2023-10-09T06:06:02.951868+0000 mgr.x (mgr.25270) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:06:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:04 smithi125 bash[27513]: audit 2023-10-09T06:06:04.290175+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.125:0/2378448016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:04 smithi155 bash[27817]: cephadm 2023-10-09T06:06:02.933902+0000 mgr.x (mgr.25270) 32 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:06:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:04 smithi155 bash[27817]: cephadm 2023-10-09T06:06:02.951868+0000 mgr.x (mgr.25270) 33 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi125 2023-10-09T06:06:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:04 smithi155 bash[27817]: audit 2023-10-09T06:06:04.290175+0000 mon.c (mon.2) 135 : audit [DBG] from='client.? 172.21.15.125:0/2378448016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:05.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:05 smithi125 bash[33919]: cluster 2023-10-09T06:06:04.331471+0000 mgr.x (mgr.25270) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:05.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:05 smithi125 bash[27513]: cluster 2023-10-09T06:06:04.331471+0000 mgr.x (mgr.25270) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:05 smithi155 bash[27817]: cluster 2023-10-09T06:06:04.331471+0000 mgr.x (mgr.25270) 34 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:07 smithi125 bash[27513]: cluster 2023-10-09T06:06:06.332723+0000 mgr.x (mgr.25270) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:07 smithi125 bash[27513]: audit 2023-10-09T06:06:06.731869+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.125:0/2515649081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:07 smithi125 bash[33919]: cluster 2023-10-09T06:06:06.332723+0000 mgr.x (mgr.25270) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:07 smithi125 bash[33919]: audit 2023-10-09T06:06:06.731869+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.125:0/2515649081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:07 smithi155 bash[27817]: cluster 2023-10-09T06:06:06.332723+0000 mgr.x (mgr.25270) 35 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-09T06:06:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:07 smithi155 bash[27817]: audit 2023-10-09T06:06:06.731869+0000 mon.c (mon.2) 136 : audit [DBG] from='client.? 172.21.15.125:0/2515649081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:09.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:08 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:06:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[27513]: cluster 2023-10-09T06:06:08.333397+0000 mgr.x (mgr.25270) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[27513]: audit 2023-10-09T06:06:08.590665+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[27513]: audit 2023-10-09T06:06:09.191225+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.125:0/1379864094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[33919]: cluster 2023-10-09T06:06:08.333397+0000 mgr.x (mgr.25270) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[33919]: audit 2023-10-09T06:06:08.590665+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:09 smithi125 bash[33919]: audit 2023-10-09T06:06:09.191225+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.125:0/1379864094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:09 smithi155 bash[27817]: cluster 2023-10-09T06:06:08.333397+0000 mgr.x (mgr.25270) 36 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:09 smithi155 bash[27817]: audit 2023-10-09T06:06:08.590665+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:09 smithi155 bash[27817]: audit 2023-10-09T06:06:09.191225+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.125:0/1379864094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:11.446 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:11 smithi125 systemd[1]: Stopping Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:11.446 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:11 smithi125 bash[72426]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-09T06:06:11.446 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:11 smithi125 bash[56935]: level=info ts=2023-10-09T06:06:11.341Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:06:11.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:11 smithi155 bash[27817]: cluster 2023-10-09T06:06:10.334505+0000 mgr.x (mgr.25270) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:11.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:11 smithi125 bash[27513]: cluster 2023-10-09T06:06:10.334505+0000 mgr.x (mgr.25270) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:11 smithi125 bash[33919]: cluster 2023-10-09T06:06:10.334505+0000 mgr.x (mgr.25270) 37 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:11 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:11] "GET /metrics HTTP/1.1" 200 33931 "" "Prometheus/2.33.4" 2023-10-09T06:06:12.228 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:11 smithi125 bash[72465]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:06:12.228 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[72551]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:06:12.229 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:12 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:06:12.229 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:12 smithi125 systemd[1]: Stopped Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:12.229 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:12 smithi125 systemd[1]: Started Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:12.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:12 smithi155 bash[48405]: ts=2023-10-09T06:06:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.125:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:12.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:12 smithi155 bash[27817]: audit 2023-10-09T06:06:11.632243+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.125:0/1371853809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:12 smithi155 bash[27817]: audit 2023-10-09T06:06:12.203014+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:12 smithi155 bash[27817]: audit 2023-10-09T06:06:12.213398+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:12.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[27513]: audit 2023-10-09T06:06:11.632243+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.125:0/1371853809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:12.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[27513]: audit 2023-10-09T06:06:12.203014+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:12.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[27513]: audit 2023-10-09T06:06:12.213398+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:12.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[33919]: audit 2023-10-09T06:06:11.632243+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.125:0/1371853809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:12.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[33919]: audit 2023-10-09T06:06:12.203014+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:12.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:12 smithi125 bash[33919]: audit 2023-10-09T06:06:12.213398+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:13.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[27513]: cephadm 2023-10-09T06:06:12.244067+0000 mgr.x (mgr.25270) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:06:13.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[27513]: cluster 2023-10-09T06:06:12.335113+0000 mgr.x (mgr.25270) 39 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:13.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[27513]: cephadm 2023-10-09T06:06:12.487319+0000 mgr.x (mgr.25270) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:06:13.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[33919]: cephadm 2023-10-09T06:06:12.244067+0000 mgr.x (mgr.25270) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:06:13.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[33919]: cluster 2023-10-09T06:06:12.335113+0000 mgr.x (mgr.25270) 39 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:13.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[33919]: cephadm 2023-10-09T06:06:12.487319+0000 mgr.x (mgr.25270) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:06:13.637 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.420Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-09T06:06:13.637 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.420Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-09T06:06:13.637 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.423Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.125 port=9094 2023-10-09T06:06:13.637 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.424Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T06:06:13.637 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.471Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:06:13.638 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.471Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:06:13.638 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.475Z caller=main.go:518 msg=Listening address=:9093 2023-10-09T06:06:13.638 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:13 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:13.475Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-09T06:06:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:13 smithi155 bash[27817]: cephadm 2023-10-09T06:06:12.244067+0000 mgr.x (mgr.25270) 38 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:06:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:13 smithi155 bash[27817]: cluster 2023-10-09T06:06:12.335113+0000 mgr.x (mgr.25270) 39 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:13 smithi155 bash[27817]: cephadm 2023-10-09T06:06:12.487319+0000 mgr.x (mgr.25270) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:06:14.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:14 smithi125 bash[33919]: audit 2023-10-09T06:06:14.068028+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.125:0/3105741341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:14.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:14 smithi125 bash[27513]: audit 2023-10-09T06:06:14.068028+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.125:0/3105741341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:14 smithi155 bash[27817]: audit 2023-10-09T06:06:14.068028+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.125:0/3105741341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:15.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:15 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:15.425Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000118922s 2023-10-09T06:06:15.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:15 smithi125 bash[27513]: cluster 2023-10-09T06:06:14.335736+0000 mgr.x (mgr.25270) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:15 smithi125 bash[33919]: cluster 2023-10-09T06:06:14.335736+0000 mgr.x (mgr.25270) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:15 smithi155 bash[27817]: cluster 2023-10-09T06:06:14.335736+0000 mgr.x (mgr.25270) 41 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:16.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:16 smithi125 bash[27513]: audit 2023-10-09T06:06:16.512652+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.125:0/2273561520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:16.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:16 smithi125 bash[33919]: audit 2023-10-09T06:06:16.512652+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.125:0/2273561520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:16 smithi155 bash[27817]: audit 2023-10-09T06:06:16.512652+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.125:0/2273561520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:17.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:17 smithi125 bash[27513]: cluster 2023-10-09T06:06:16.336485+0000 mgr.x (mgr.25270) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:17 smithi125 bash[33919]: cluster 2023-10-09T06:06:16.336485+0000 mgr.x (mgr.25270) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:17 smithi155 bash[27817]: cluster 2023-10-09T06:06:16.336485+0000 mgr.x (mgr.25270) 42 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:19.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:18 smithi125 bash[27779]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:06:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:19 smithi125 bash[27513]: cluster 2023-10-09T06:06:18.337121+0000 mgr.x (mgr.25270) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:19.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:19 smithi125 bash[27513]: audit 2023-10-09T06:06:18.965200+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.125:0/101714353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:19 smithi125 bash[33919]: cluster 2023-10-09T06:06:18.337121+0000 mgr.x (mgr.25270) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:19 smithi125 bash[33919]: audit 2023-10-09T06:06:18.965200+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.125:0/101714353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:19 smithi155 bash[27817]: cluster 2023-10-09T06:06:18.337121+0000 mgr.x (mgr.25270) 43 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:19 smithi155 bash[27817]: audit 2023-10-09T06:06:18.965200+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.125:0/101714353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:21.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:21 smithi155 bash[27817]: cluster 2023-10-09T06:06:20.338191+0000 mgr.x (mgr.25270) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:21.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:21 smithi155 bash[27817]: audit 2023-10-09T06:06:21.404152+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.125:0/1662753474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:21.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:21 smithi125 bash[27513]: cluster 2023-10-09T06:06:20.338191+0000 mgr.x (mgr.25270) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:21.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:21 smithi125 bash[27513]: audit 2023-10-09T06:06:21.404152+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.125:0/1662753474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:21 smithi125 bash[33919]: cluster 2023-10-09T06:06:20.338191+0000 mgr.x (mgr.25270) 44 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:21 smithi125 bash[33919]: audit 2023-10-09T06:06:21.404152+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.125:0/1662753474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:22.140 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:21 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:21] "GET /metrics HTTP/1.1" 200 33931 "" "Prometheus/2.33.4" 2023-10-09T06:06:22.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:22 smithi155 bash[48405]: ts=2023-10-09T06:06:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.125:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:23.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:23 smithi125 bash[27513]: cluster 2023-10-09T06:06:22.338780+0000 mgr.x (mgr.25270) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:23 smithi125 bash[33919]: cluster 2023-10-09T06:06:22.338780+0000 mgr.x (mgr.25270) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:23.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:06:23 smithi125 bash[72578]: level=info ts=2023-10-09T06:06:23.429Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004464363s 2023-10-09T06:06:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:23 smithi155 bash[27817]: cluster 2023-10-09T06:06:22.338780+0000 mgr.x (mgr.25270) 45 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:24.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 systemd[1]: Stopping Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:24.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[52846]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T06:06:24.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.274Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.275Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:06:24.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.275Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T06:06:24.395 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.275Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:06:24.395 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.276Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:06:24.395 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.276Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T06:06:24.395 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[48405]: ts=2023-10-09T06:06:24.276Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T06:06:24.767 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[27817]: audit 2023-10-09T06:06:23.590278+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:24.768 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[27817]: audit 2023-10-09T06:06:23.836547+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.125:0/1527643339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:24.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:24 smithi125 bash[27513]: audit 2023-10-09T06:06:23.590278+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:24.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:24 smithi125 bash[27513]: audit 2023-10-09T06:06:23.836547+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.125:0/1527643339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:24.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:24 smithi125 bash[33919]: audit 2023-10-09T06:06:23.590278+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:24.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:24 smithi125 bash[33919]: audit 2023-10-09T06:06:23.836547+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.125:0/1527643339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:25.056 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[52882]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:06:25.056 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 bash[52948]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:06:25.056 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:06:25.056 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 systemd[1]: Stopped Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:25.056 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:24 smithi155 systemd[1]: Started Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:25.643 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:25 smithi155 systemd[1]: Stopping Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:25.643 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:25 smithi155 bash[53001]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter.b 2023-10-09T06:06:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:25 smithi155 bash[27817]: cluster 2023-10-09T06:06:24.339417+0000 mgr.x (mgr.25270) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:25 smithi155 bash[27817]: audit 2023-10-09T06:06:25.026865+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:25 smithi155 bash[27817]: audit 2023-10-09T06:06:25.035800+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[27513]: cluster 2023-10-09T06:06:24.339417+0000 mgr.x (mgr.25270) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[27513]: audit 2023-10-09T06:06:25.026865+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[27513]: audit 2023-10-09T06:06:25.035800+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[33919]: cluster 2023-10-09T06:06:24.339417+0000 mgr.x (mgr.25270) 46 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[33919]: audit 2023-10-09T06:06:25.026865+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:25 smithi125 bash[33919]: audit 2023-10-09T06:06:25.035800+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:26.828 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[27817]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:06:26.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[27817]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:06:26.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[27817]: audit 2023-10-09T06:06:26.282714+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.125:0/1513191958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:26.829 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.815Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T06:06:26.829 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.815Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T06:06:26.829 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.815Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T06:06:26.829 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.815Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.815Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.817Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.818Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.819Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.822Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.824Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.219456ms 2023-10-09T06:06:26.830 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.824Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T06:06:26.830 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[53037]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T06:06:26.831 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:06:26.831 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[53171]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T06:06:26.831 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:06:26.831 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 systemd[1]: Stopped Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:26.831 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:26 smithi155 systemd[1]: Started Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[27513]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[27513]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[27513]: audit 2023-10-09T06:06:26.282714+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.125:0/1513191958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[33919]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[33919]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:06:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:26 smithi125 bash[33919]: audit 2023-10-09T06:06:26.282714+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.125:0/1513191958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:27.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:26 smithi155 bash[52978]: ts=2023-10-09T06:06:26.982Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.257Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.257Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.258Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=45.323µs wal_replay_duration=433.728911ms total_replay_duration=435.015204ms 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.263Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.263Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T06:06:27.609 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.263Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T06:06:27.610 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.286Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.096094ms db_storage=1.122µs remote_storage=1.873µs web_handler=572ns query_engine=929ns scrape=360.328µs scrape_sd=124.998µs notify=18.368µs notify_sd=15.231µs rules=22.171404ms 2023-10-09T06:06:27.610 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[52978]: ts=2023-10-09T06:06:27.286Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T06:06:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[27513]: cluster 2023-10-09T06:06:26.340495+0000 mgr.x (mgr.25270) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[27513]: audit 2023-10-09T06:06:26.860547+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[27513]: audit 2023-10-09T06:06:26.869427+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[33919]: cluster 2023-10-09T06:06:26.340495+0000 mgr.x (mgr.25270) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[33919]: audit 2023-10-09T06:06:26.860547+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:27 smithi125 bash[33919]: audit 2023-10-09T06:06:26.869427+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[27817]: cluster 2023-10-09T06:06:26.340495+0000 mgr.x (mgr.25270) 49 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[27817]: audit 2023-10-09T06:06:26.860547+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:27 smithi155 bash[27817]: audit 2023-10-09T06:06:26.869427+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.143Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.143Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.144Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.144Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=arp 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-09T06:06:28.394 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=edac 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-09T06:06:28.395 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-09T06:06:28.396 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=os 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=stat 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-09T06:06:28.397 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=time 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=uname 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.145Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-09T06:06:28.398 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[53201]: ts=2023-10-09T06:06:28.146Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-09T06:06:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:28 smithi125 bash[27513]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:06:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:28 smithi125 bash[27513]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:06:28.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:28 smithi125 bash[33919]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:06:28.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:28 smithi125 bash[33919]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:06:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[27817]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:06:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:28 smithi155 bash[27817]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm [INF] Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:06:29.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:29 smithi125 bash[27513]: cluster 2023-10-09T06:06:28.341160+0000 mgr.x (mgr.25270) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:29.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:29 smithi125 bash[27513]: audit 2023-10-09T06:06:28.727324+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.125:0/3215959926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:29 smithi125 bash[33919]: cluster 2023-10-09T06:06:28.341160+0000 mgr.x (mgr.25270) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:29 smithi125 bash[33919]: audit 2023-10-09T06:06:28.727324+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.125:0/3215959926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:29 smithi155 bash[27817]: cluster 2023-10-09T06:06:28.341160+0000 mgr.x (mgr.25270) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:29 smithi155 bash[27817]: audit 2023-10-09T06:06:28.727324+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.125:0/3215959926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:31.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:31 smithi125 bash[27513]: cluster 2023-10-09T06:06:30.342112+0000 mgr.x (mgr.25270) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:31.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:31 smithi125 bash[27513]: audit 2023-10-09T06:06:31.171946+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.125:0/3706903480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:31 smithi125 bash[33919]: cluster 2023-10-09T06:06:30.342112+0000 mgr.x (mgr.25270) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:31 smithi125 bash[33919]: audit 2023-10-09T06:06:31.171946+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.125:0/3706903480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:31 smithi155 bash[27817]: cluster 2023-10-09T06:06:30.342112+0000 mgr.x (mgr.25270) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:31 smithi155 bash[27817]: audit 2023-10-09T06:06:31.171946+0000 mon.a (mon.0) 820 : audit [DBG] from='client.? 172.21.15.125:0/3706903480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:32.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:32 smithi125 bash[27513]: cluster 2023-10-09T06:06:32.342632+0000 mgr.x (mgr.25270) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:32.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:32 smithi125 bash[33919]: cluster 2023-10-09T06:06:32.342632+0000 mgr.x (mgr.25270) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:32.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:32 smithi155 bash[27817]: cluster 2023-10-09T06:06:32.342632+0000 mgr.x (mgr.25270) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:33 smithi155 bash[27817]: audit 2023-10-09T06:06:33.617644+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.125:0/1703015919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:34.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:33 smithi125 bash[27513]: audit 2023-10-09T06:06:33.617644+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.125:0/1703015919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:34.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:33 smithi125 bash[33919]: audit 2023-10-09T06:06:33.617644+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.125:0/1703015919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:35.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:34 smithi125 bash[33919]: cluster 2023-10-09T06:06:34.343137+0000 mgr.x (mgr.25270) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:35.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:34 smithi125 bash[27513]: cluster 2023-10-09T06:06:34.343137+0000 mgr.x (mgr.25270) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:35.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:34 smithi155 bash[27817]: cluster 2023-10-09T06:06:34.343137+0000 mgr.x (mgr.25270) 55 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:36.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:36 smithi125 bash[27513]: audit 2023-10-09T06:06:36.064798+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.125:0/2909832122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:36.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:36 smithi125 bash[33919]: audit 2023-10-09T06:06:36.064798+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.125:0/2909832122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:36 smithi155 bash[27817]: audit 2023-10-09T06:06:36.064798+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.125:0/2909832122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:37 smithi125 bash[27513]: cluster 2023-10-09T06:06:36.344049+0000 mgr.x (mgr.25270) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:37 smithi125 bash[33919]: cluster 2023-10-09T06:06:36.344049+0000 mgr.x (mgr.25270) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:37.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:37 smithi155 bash[27817]: cluster 2023-10-09T06:06:36.344049+0000 mgr.x (mgr.25270) 56 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:39.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:39 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:39] "GET /metrics HTTP/1.1" 200 33927 "" "Prometheus/2.33.4" 2023-10-09T06:06:39.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:39 smithi155 bash[27817]: cluster 2023-10-09T06:06:38.344628+0000 mgr.x (mgr.25270) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:39.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:39 smithi155 bash[27817]: audit 2023-10-09T06:06:38.506658+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.125:0/1826069227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:39.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:39 smithi155 bash[27817]: audit 2023-10-09T06:06:38.593521+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[27513]: cluster 2023-10-09T06:06:38.344628+0000 mgr.x (mgr.25270) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[27513]: audit 2023-10-09T06:06:38.506658+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.125:0/1826069227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[27513]: audit 2023-10-09T06:06:38.593521+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[33919]: cluster 2023-10-09T06:06:38.344628+0000 mgr.x (mgr.25270) 57 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[33919]: audit 2023-10-09T06:06:38.506658+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.125:0/1826069227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:39 smithi125 bash[33919]: audit 2023-10-09T06:06:38.593521+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:41.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:41 smithi125 bash[33919]: cluster 2023-10-09T06:06:40.345889+0000 mgr.x (mgr.25270) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:41.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:41 smithi125 bash[33919]: audit 2023-10-09T06:06:40.951962+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.125:0/3289396202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:41 smithi125 bash[27513]: cluster 2023-10-09T06:06:40.345889+0000 mgr.x (mgr.25270) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:41 smithi125 bash[27513]: audit 2023-10-09T06:06:40.951962+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.125:0/3289396202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:41.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:41 smithi155 bash[27817]: cluster 2023-10-09T06:06:40.345889+0000 mgr.x (mgr.25270) 58 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:41.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:41 smithi155 bash[27817]: audit 2023-10-09T06:06:40.951962+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.125:0/3289396202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:43.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:43 smithi125 bash[27513]: cluster 2023-10-09T06:06:42.346488+0000 mgr.x (mgr.25270) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:43.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:43 smithi125 bash[27513]: audit 2023-10-09T06:06:43.401112+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.125:0/3948042968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:43 smithi125 bash[33919]: cluster 2023-10-09T06:06:42.346488+0000 mgr.x (mgr.25270) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:43 smithi125 bash[33919]: audit 2023-10-09T06:06:43.401112+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.125:0/3948042968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:43 smithi155 bash[27817]: cluster 2023-10-09T06:06:42.346488+0000 mgr.x (mgr.25270) 59 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:43 smithi155 bash[27817]: audit 2023-10-09T06:06:43.401112+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.125:0/3948042968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:45 smithi125 bash[27513]: cluster 2023-10-09T06:06:44.346927+0000 mgr.x (mgr.25270) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:45 smithi125 bash[33919]: cluster 2023-10-09T06:06:44.346927+0000 mgr.x (mgr.25270) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:45 smithi155 bash[27817]: cluster 2023-10-09T06:06:44.346927+0000 mgr.x (mgr.25270) 60 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:46 smithi125 bash[27513]: audit 2023-10-09T06:06:45.853248+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.125:0/1695826346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:46 smithi125 bash[33919]: audit 2023-10-09T06:06:45.853248+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.125:0/1695826346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:46 smithi155 bash[27817]: audit 2023-10-09T06:06:45.853248+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.125:0/1695826346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:47 smithi125 bash[27513]: cluster 2023-10-09T06:06:46.347960+0000 mgr.x (mgr.25270) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:47 smithi125 bash[33919]: cluster 2023-10-09T06:06:46.347960+0000 mgr.x (mgr.25270) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:47 smithi155 bash[27817]: cluster 2023-10-09T06:06:46.347960+0000 mgr.x (mgr.25270) 61 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:48.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:48 smithi125 bash[27513]: audit 2023-10-09T06:06:48.302047+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.125:0/2851358610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:48.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:48 smithi125 bash[33919]: audit 2023-10-09T06:06:48.302047+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.125:0/2851358610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:48.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:48 smithi155 bash[52978]: ts=2023-10-09T06:06:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:48.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:48 smithi155 bash[27817]: audit 2023-10-09T06:06:48.302047+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.125:0/2851358610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:49 smithi155 bash[27817]: cluster 2023-10-09T06:06:48.348527+0000 mgr.x (mgr.25270) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:49.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:49 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:49] "GET /metrics HTTP/1.1" 200 33927 "" "Prometheus/2.33.4" 2023-10-09T06:06:49.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:49 smithi125 bash[27513]: cluster 2023-10-09T06:06:48.348527+0000 mgr.x (mgr.25270) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:49.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:49 smithi125 bash[33919]: cluster 2023-10-09T06:06:48.348527+0000 mgr.x (mgr.25270) 62 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:51.248 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:50 smithi155 systemd[1]: Stopping Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:51.248 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:50 smithi155 bash[53449]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-grafana.a 2023-10-09T06:06:51.248 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[47697]: t=2023-10-09T06:06:51+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T06:06:51.248 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[47697]: t=2023-10-09T06:06:51+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-10-09T06:06:51.730 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[27817]: cluster 2023-10-09T06:06:50.349662+0000 mgr.x (mgr.25270) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:51.730 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[27817]: audit 2023-10-09T06:06:50.739233+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.125:0/1269646436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:51.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:51 smithi125 bash[27513]: cluster 2023-10-09T06:06:50.349662+0000 mgr.x (mgr.25270) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:51.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:51 smithi125 bash[27513]: audit 2023-10-09T06:06:50.739233+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.125:0/1269646436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:51 smithi125 bash[33919]: cluster 2023-10-09T06:06:50.349662+0000 mgr.x (mgr.25270) 63 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:51 smithi125 bash[33919]: audit 2023-10-09T06:06:50.739233+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.125:0/1269646436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:51.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[53484]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T06:06:52.393 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 bash[53548]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T06:06:52.393 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T06:06:52.393 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 systemd[1]: Stopped Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:52.393 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:51 smithi155 systemd[1]: Started Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.011346+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.020130+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.047556+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.048251+0000 mgr.x (mgr.25270) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.052103+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.052593+0000 mgr.x (mgr.25270) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.056073+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.056581+0000 mgr.x (mgr.25270) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: audit 2023-10-09T06:06:52.063213+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:53.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[27817]: cluster 2023-10-09T06:06:52.350312+0000 mgr.x (mgr.25270) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:53.353 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-09T06:06:53.353 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-09T06:06:53.353 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-09T06:06:53.353 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-09T06:06:53.354 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="App mode production" logger=settings 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.525361ms 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-09T06:06:53.355 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-09T06:06:53.356 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T06:06:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.011346+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.020130+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.047556+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.048251+0000 mgr.x (mgr.25270) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.052103+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.052593+0000 mgr.x (mgr.25270) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.056073+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.056581+0000 mgr.x (mgr.25270) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: audit 2023-10-09T06:06:52.063213+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[33919]: cluster 2023-10-09T06:06:52.350312+0000 mgr.x (mgr.25270) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.011346+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.020130+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:53.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.047556+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.048251+0000 mgr.x (mgr.25270) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.052103+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.052593+0000 mgr.x (mgr.25270) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.056073+0000 mon.b (mon.1) 80 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.056581+0000 mgr.x (mgr.25270) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: audit 2023-10-09T06:06:52.063213+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:06:53.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:53 smithi125 bash[27513]: cluster 2023-10-09T06:06:52.350312+0000 mgr.x (mgr.25270) 69 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:53.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-09T06:06:53.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-09T06:06:53.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-09T06:06:53.643 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:06:53 smithi155 bash[53577]: t=2023-10-09T06:06:53+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T06:06:54.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:54 smithi125 bash[27513]: audit 2023-10-09T06:06:53.180365+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.125:0/2811600522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:54.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:54 smithi125 bash[27513]: audit 2023-10-09T06:06:53.594702+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:54.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:54 smithi125 bash[33919]: audit 2023-10-09T06:06:53.180365+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.125:0/2811600522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:54.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:54 smithi125 bash[33919]: audit 2023-10-09T06:06:53.594702+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:54 smithi155 bash[27817]: audit 2023-10-09T06:06:53.180365+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.125:0/2811600522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:54 smithi155 bash[27817]: audit 2023-10-09T06:06:53.594702+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:55.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: cluster 2023-10-09T06:06:54.350974+0000 mgr.x (mgr.25270) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:55.788 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: audit 2023-10-09T06:06:54.496603+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: audit 2023-10-09T06:06:54.499553+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: audit 2023-10-09T06:06:54.499829+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: audit 2023-10-09T06:06:54.502237+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: audit 2023-10-09T06:06:54.503582+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[33919]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm [INF] Deploying daemon mgr.y on smithi125 2023-10-09T06:06:55.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: cluster 2023-10-09T06:06:54.350974+0000 mgr.x (mgr.25270) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:55.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T06:06:55.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: audit 2023-10-09T06:06:54.496603+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:55.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: audit 2023-10-09T06:06:54.499553+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: audit 2023-10-09T06:06:54.499829+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: audit 2023-10-09T06:06:54.502237+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:55.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: audit 2023-10-09T06:06:54.503582+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:55 smithi125 bash[27513]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm [INF] Deploying daemon mgr.y on smithi125 2023-10-09T06:06:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: cluster 2023-10-09T06:06:54.350974+0000 mgr.x (mgr.25270) 70 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-09T06:06:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: audit 2023-10-09T06:06:54.496603+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: audit 2023-10-09T06:06:54.499553+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: audit 2023-10-09T06:06:54.499829+0000 mon.b (mon.1) 83 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:55.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: audit 2023-10-09T06:06:54.502237+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:55.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: audit 2023-10-09T06:06:54.503582+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:55 smithi155 bash[27817]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm [INF] Deploying daemon mgr.y on smithi125 2023-10-09T06:06:56.785 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:56 smithi125 bash[27513]: audit 2023-10-09T06:06:55.618619+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.125:0/3630250993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:56.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:56 smithi125 bash[33919]: audit 2023-10-09T06:06:55.618619+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.125:0/3630250993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:56.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:56 smithi155 bash[27817]: audit 2023-10-09T06:06:55.618619+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.125:0/3630250993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:57.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 systemd[1]: Stopping Ceph mgr.y for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:57.764 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:57 smithi125 bash[27513]: cluster 2023-10-09T06:06:56.352073+0000 mgr.x (mgr.25270) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:57.764 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:57 smithi125 bash[33919]: cluster 2023-10-09T06:06:56.352073+0000 mgr.x (mgr.25270) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:57 smithi155 bash[27817]: cluster 2023-10-09T06:06:56.352073+0000 mgr.x (mgr.25270) 73 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:58.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 bash[73338]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T06:06:58.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:06:58.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 bash[73423]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T06:06:58.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T06:06:58.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:57 smithi125 systemd[1]: Stopped Ceph mgr.y for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:58.506 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:58 smithi125 systemd[1]: Started Ceph mgr.y for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[33919]: audit 2023-10-09T06:06:58.067611+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.125:0/414418253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[33919]: audit 2023-10-09T06:06:58.237994+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[33919]: audit 2023-10-09T06:06:58.248499+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[33919]: audit 2023-10-09T06:06:58.279570+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[27513]: audit 2023-10-09T06:06:58.067611+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.125:0/414418253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[27513]: audit 2023-10-09T06:06:58.237994+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[27513]: audit 2023-10-09T06:06:58.248499+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:58 smithi125 bash[27513]: audit 2023-10-09T06:06:58.279570+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:58.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:58 smithi155 bash[27817]: audit 2023-10-09T06:06:58.067611+0000 mon.a (mon.0) 826 : audit [DBG] from='client.? 172.21.15.125:0/414418253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:58.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:58 smithi155 bash[27817]: audit 2023-10-09T06:06:58.237994+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:58 smithi155 bash[27817]: audit 2023-10-09T06:06:58.248499+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:06:58.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:58 smithi155 bash[27817]: audit 2023-10-09T06:06:58.279570+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:58.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:06:58 smithi155 bash[52978]: ts=2023-10-09T06:06:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:59.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:06:59 smithi125 bash[27513]: cluster 2023-10-09T06:06:58.352578+0000 mgr.x (mgr.25270) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:59.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:06:59 smithi125 bash[33919]: cluster 2023-10-09T06:06:58.352578+0000 mgr.x (mgr.25270) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:06:59.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:59 smithi125 bash[73481]: debug 2023-10-09T06:06:59.495+0000 7fa7b7e99700 1 -- 172.21.15.125:0/728377388 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562f064f51e0 con 0x562f07274400 2023-10-09T06:06:59.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:06:59 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:06:59] "GET /metrics HTTP/1.1" 200 33928 "" "Prometheus/2.33.4" 2023-10-09T06:06:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:06:59 smithi155 bash[27817]: cluster 2023-10-09T06:06:58.352578+0000 mgr.x (mgr.25270) 74 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:00.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:06:59 smithi125 bash[73481]: debug 2023-10-09T06:06:59.663+0000 7fa7c1fe7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:07:00.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:00 smithi125 bash[73481]: debug 2023-10-09T06:07:00.591+0000 7fa7c1fe7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:07:00.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:00 smithi125 bash[73481]: debug 2023-10-09T06:07:00.695+0000 7fa7c1fe7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:07:00.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:00 smithi125 bash[73481]: debug 2023-10-09T06:07:00.799+0000 7fa7c1fe7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:07:01.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[73481]: debug 2023-10-09T06:07:01.063+0000 7fa7c1fe7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:01.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[27513]: cluster 2023-10-09T06:07:00.353588+0000 mgr.x (mgr.25270) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:01.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[27513]: audit 2023-10-09T06:07:00.510657+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.125:0/621100685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:01.816 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[73481]: debug 2023-10-09T06:07:01.427+0000 7fa7c1fe7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:07:01.816 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[73481]: debug 2023-10-09T06:07:01.587+0000 7fa7c1fe7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:01.816 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[33919]: cluster 2023-10-09T06:07:00.353588+0000 mgr.x (mgr.25270) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:01.816 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[33919]: audit 2023-10-09T06:07:00.510657+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.125:0/621100685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:01 smithi155 bash[27817]: cluster 2023-10-09T06:07:00.353588+0000 mgr.x (mgr.25270) 75 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:01 smithi155 bash[27817]: audit 2023-10-09T06:07:00.510657+0000 mon.a (mon.0) 829 : audit [DBG] from='client.? 172.21.15.125:0/621100685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:02.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:01 smithi125 bash[73481]: debug 2023-10-09T06:07:01.807+0000 7fa7c1fe7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:07:03.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:02 smithi125 bash[73481]: debug 2023-10-09T06:07:02.843+0000 7fa7c1fe7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:07:03.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:02 smithi125 bash[73481]: debug 2023-10-09T06:07:02.963+0000 7fa7c1fe7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:07:03.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[73481]: debug 2023-10-09T06:07:03.087+0000 7fa7c1fe7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:07:03.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[33919]: cluster 2023-10-09T06:07:02.354361+0000 mgr.x (mgr.25270) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:03.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[33919]: audit 2023-10-09T06:07:02.992192+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.125:0/72017545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:03.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[27513]: cluster 2023-10-09T06:07:02.354361+0000 mgr.x (mgr.25270) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:03.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[27513]: audit 2023-10-09T06:07:02.992192+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.125:0/72017545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:03.873 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[73481]: debug 2023-10-09T06:07:03.607+0000 7fa7c1fe7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:07:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:03 smithi155 bash[27817]: cluster 2023-10-09T06:07:02.354361+0000 mgr.x (mgr.25270) 76 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:03 smithi155 bash[27817]: audit 2023-10-09T06:07:02.992192+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.125:0/72017545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:04.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:03 smithi125 bash[73481]: debug 2023-10-09T06:07:03.863+0000 7fa7c1fe7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:07:04.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:04 smithi125 bash[73481]: debug 2023-10-09T06:07:04.403+0000 7fa7c1fe7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:07:05.275 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:04 smithi125 bash[73481]: debug 2023-10-09T06:07:04.955+0000 7fa7c1fe7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:04.273205+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:04.281266+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: cluster 2023-10-09T06:07:04.354872+0000 mgr.x (mgr.25270) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:04.582972+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:04.594887+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:05.140767+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.608 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[33919]: audit 2023-10-09T06:07:05.150106+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:04.273205+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:04.281266+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: cluster 2023-10-09T06:07:04.354872+0000 mgr.x (mgr.25270) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:04.582972+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:04.594887+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:05.140767+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[27513]: audit 2023-10-09T06:07:05.150106+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:04.273205+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:04.281266+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: cluster 2023-10-09T06:07:04.354872+0000 mgr.x (mgr.25270) 77 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:04.582972+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:04.594887+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:05.140767+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:05 smithi155 bash[27817]: audit 2023-10-09T06:07:05.150106+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:05.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[73481]: debug 2023-10-09T06:07:05.599+0000 7fa7c1fe7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:07:05.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:05 smithi125 bash[73481]: debug 2023-10-09T06:07:05.727+0000 7fa7c1fe7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:07:06.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[27513]: audit 2023-10-09T06:07:05.434080+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.125:0/1635998484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:06.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[33919]: audit 2023-10-09T06:07:05.434080+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.125:0/1635998484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:06.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.119+0000 7fa7c1fe7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:07:06.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.239+0000 7fa7c1fe7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:07:06.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:06 smithi155 bash[27817]: audit 2023-10-09T06:07:05.434080+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.125:0/1635998484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.387+0000 7fa7c1fe7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.511+0000 7fa7c1fe7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.643+0000 7fa7c1fe7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:07:06.973 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.839+0000 7fa7c1fe7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:07:06.973 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:06 smithi125 bash[73481]: debug 2023-10-09T06:07:06.967+0000 7fa7c1fe7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:07:07.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:07 smithi125 bash[27513]: cluster 2023-10-09T06:07:06.355934+0000 mgr.x (mgr.25270) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:07.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:07 smithi125 bash[33919]: cluster 2023-10-09T06:07:06.355934+0000 mgr.x (mgr.25270) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:07.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:07 smithi155 bash[27817]: cluster 2023-10-09T06:07:06.355934+0000 mgr.x (mgr.25270) 78 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:08.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: debug 2023-10-09T06:07:08.235+0000 7fa7c1fe7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:07:08.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: [09/Oct/2023:06:07:08] ENGINE Bus STARTING 2023-10-09T06:07:08.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: CherryPy Checker: 2023-10-09T06:07:08.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: The Application mounted at '' has an empty config. 2023-10-09T06:07:08.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: [09/Oct/2023:06:07:08] ENGINE Serving on http://:::9283 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[73481]: [09/Oct/2023:06:07:08] ENGINE Bus STARTED 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: audit 2023-10-09T06:07:07.868176+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.125:0/3025089622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: cluster 2023-10-09T06:07:08.261115+0000 mon.a (mon.0) 837 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: cluster 2023-10-09T06:07:08.261353+0000 mon.a (mon.0) 838 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: audit 2023-10-09T06:07:08.266231+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: audit 2023-10-09T06:07:08.267129+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: audit 2023-10-09T06:07:08.272259+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[27513]: audit 2023-10-09T06:07:08.273864+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: audit 2023-10-09T06:07:07.868176+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.125:0/3025089622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: cluster 2023-10-09T06:07:08.261115+0000 mon.a (mon.0) 837 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: cluster 2023-10-09T06:07:08.261353+0000 mon.a (mon.0) 838 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: audit 2023-10-09T06:07:08.266231+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: audit 2023-10-09T06:07:08.267129+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: audit 2023-10-09T06:07:08.272259+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:07:08.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:08 smithi125 bash[33919]: audit 2023-10-09T06:07:08.273864+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: audit 2023-10-09T06:07:07.868176+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.125:0/3025089622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: cluster 2023-10-09T06:07:08.261115+0000 mon.a (mon.0) 837 : cluster [DBG] Standby manager daemon y restarted 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: cluster 2023-10-09T06:07:08.261353+0000 mon.a (mon.0) 838 : cluster [DBG] Standby manager daemon y started 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: audit 2023-10-09T06:07:08.266231+0000 mon.a (mon.0) 839 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: audit 2023-10-09T06:07:08.267129+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: audit 2023-10-09T06:07:08.272259+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:07:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[27817]: audit 2023-10-09T06:07:08.273864+0000 mon.a (mon.0) 842 : audit [DBG] from='mgr.? 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:09.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:08 smithi155 bash[52978]: ts=2023-10-09T06:07:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:07:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[27513]: cluster 2023-10-09T06:07:08.304219+0000 mon.a (mon.0) 843 : cluster [DBG] mgrmap e24: x(active, since 89s), standbys: y 2023-10-09T06:07:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[27513]: cluster 2023-10-09T06:07:08.356568+0000 mgr.x (mgr.25270) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[27513]: audit 2023-10-09T06:07:08.595113+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[33919]: cluster 2023-10-09T06:07:08.304219+0000 mon.a (mon.0) 843 : cluster [DBG] mgrmap e24: x(active, since 89s), standbys: y 2023-10-09T06:07:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[33919]: cluster 2023-10-09T06:07:08.356568+0000 mgr.x (mgr.25270) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:09 smithi125 bash[33919]: audit 2023-10-09T06:07:08.595113+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:09.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:09 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:07:09] "GET /metrics HTTP/1.1" 200 33926 "" "Prometheus/2.33.4" 2023-10-09T06:07:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:09 smithi155 bash[27817]: cluster 2023-10-09T06:07:08.304219+0000 mon.a (mon.0) 843 : cluster [DBG] mgrmap e24: x(active, since 89s), standbys: y 2023-10-09T06:07:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:09 smithi155 bash[27817]: cluster 2023-10-09T06:07:08.356568+0000 mgr.x (mgr.25270) 79 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:09 smithi155 bash[27817]: audit 2023-10-09T06:07:08.595113+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:10.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:10 smithi125 bash[33919]: cluster 2023-10-09T06:07:09.327611+0000 mon.a (mon.0) 844 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T06:07:10.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:10 smithi125 bash[33919]: audit 2023-10-09T06:07:10.314726+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.125:0/1263695013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:10.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:10 smithi125 bash[27513]: cluster 2023-10-09T06:07:09.327611+0000 mon.a (mon.0) 844 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T06:07:10.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:10 smithi125 bash[27513]: audit 2023-10-09T06:07:10.314726+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.125:0/1263695013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:10.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:10 smithi155 bash[27817]: cluster 2023-10-09T06:07:09.327611+0000 mon.a (mon.0) 844 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-10-09T06:07:10.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:10 smithi155 bash[27817]: audit 2023-10-09T06:07:10.314726+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.125:0/1263695013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:11.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:11 smithi125 bash[27513]: cluster 2023-10-09T06:07:10.357778+0000 mgr.x (mgr.25270) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:11 smithi125 bash[33919]: cluster 2023-10-09T06:07:10.357778+0000 mgr.x (mgr.25270) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:11 smithi155 bash[27817]: cluster 2023-10-09T06:07:10.357778+0000 mgr.x (mgr.25270) 80 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:13.763 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:13 smithi125 bash[27513]: cluster 2023-10-09T06:07:12.358435+0000 mgr.x (mgr.25270) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:13.763 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:13 smithi125 bash[27513]: audit 2023-10-09T06:07:12.770809+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.125:0/1610241962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:13.764 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:13 smithi125 bash[33919]: cluster 2023-10-09T06:07:12.358435+0000 mgr.x (mgr.25270) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:13.764 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:13 smithi125 bash[33919]: audit 2023-10-09T06:07:12.770809+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.125:0/1610241962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:13 smithi155 bash[27817]: cluster 2023-10-09T06:07:12.358435+0000 mgr.x (mgr.25270) 81 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:13 smithi155 bash[27817]: audit 2023-10-09T06:07:12.770809+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.125:0/1610241962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:14.969 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE Bus STOPPING 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE Bus STOPPED 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE Bus STARTING 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE Serving on http://:::9283 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: [09/Oct/2023:06:07:14] ENGINE Bus STARTED 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71e47f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71e47f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71e47f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71e47f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.970 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71e47f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71b479700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71b479700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71b479700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.931+0000 7fa71b479700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.971 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:14 smithi125 bash[73481]: debug 2023-10-09T06:07:14.935+0000 7fa71b479700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:14 smithi155 bash[50569]: debug 2023-10-09T06:07:14.683+0000 7eff29c04700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T06:07:14.972 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:14 smithi155 bash[50569]: ignoring --setuser ceph since I am not root 2023-10-09T06:07:14.973 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:14 smithi155 bash[50569]: ignoring --setgroup ceph since I am not root 2023-10-09T06:07:14.973 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:14 smithi155 bash[50569]: debug 2023-10-09T06:07:14.743+0000 7f97b1104700 1 -- 172.21.15.155:0/2133697086 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a8709211e0 con 0x55a8716a0400 2023-10-09T06:07:14.973 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:14 smithi155 bash[50569]: debug 2023-10-09T06:07:14.907+0000 7f97bae34200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:07:15.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: cluster 2023-10-09T06:07:14.359165+0000 mgr.x (mgr.25270) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:15.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.553349+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.562406+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.567344+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.568821+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.576694+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.662528+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.674003+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.674335+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: cluster 2023-10-09T06:07:14.685242+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.685715+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: cluster 2023-10-09T06:07:14.685830+0000 mon.a (mon.0) 852 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0109123s) 2023-10-09T06:07:15.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.687972+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.688201+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.688460+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.688974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.689295+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.690133+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.690385+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.690790+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.691117+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.691491+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.691833+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:07:15.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.692199+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.692561+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.692933+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.693319+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.693752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.694433+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: cluster 2023-10-09T06:07:14.745997+0000 mon.a (mon.0) 853 : cluster [INF] Manager daemon y is now available 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.782100+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.785038+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.821838+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:15.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.837984+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.838722+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.885406+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:14.886325+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[27817]: audit 2023-10-09T06:07:15.243236+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.125:0/3539121397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:15.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: cluster 2023-10-09T06:07:14.359165+0000 mgr.x (mgr.25270) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:15.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.553349+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.562406+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.567344+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.568821+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.576694+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.662528+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.674003+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.674335+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: cluster 2023-10-09T06:07:14.685242+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.685715+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: cluster 2023-10-09T06:07:14.685830+0000 mon.a (mon.0) 852 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0109123s) 2023-10-09T06:07:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.687972+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.688201+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.688460+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.688974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.689295+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.690133+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.690385+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.690790+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:07:15.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: cluster 2023-10-09T06:07:14.359165+0000 mgr.x (mgr.25270) 82 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.553349+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.562406+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.567344+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.568821+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.576694+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.25270 ' entity='mgr.x' 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.662528+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.674003+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.674335+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25270 172.21.15.155:0/3483222038' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: cluster 2023-10-09T06:07:14.685242+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.685715+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25270 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:07:15.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: cluster 2023-10-09T06:07:14.685830+0000 mon.a (mon.0) 852 : cluster [DBG] mgrmap e26: y(active, starting, since 0.0109123s) 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.687972+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.688201+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.688460+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.688974+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.689295+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.690133+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.690385+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.690790+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:07:15.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.691117+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.691491+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.691833+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.692199+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.692561+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.692933+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.693319+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.693752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.694433+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: cluster 2023-10-09T06:07:14.745997+0000 mon.a (mon.0) 853 : cluster [INF] Manager daemon y is now available 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.782100+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.785038+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.821838+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.837984+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.838722+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.885406+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:14.886325+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[27513]: audit 2023-10-09T06:07:15.243236+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.125:0/3539121397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.691117+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.691491+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:07:15.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.691833+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.692199+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.692561+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.692933+0000 mon.c (mon.2) 167 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.693319+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.693752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.694433+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: cluster 2023-10-09T06:07:14.745997+0000 mon.a (mon.0) 853 : cluster [INF] Manager daemon y is now available 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.782100+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.785038+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.821838+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.837984+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.838722+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:07:15.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.885406+0000 mon.c (mon.2) 175 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:14.886325+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:07:15.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:15 smithi125 bash[33919]: audit 2023-10-09T06:07:15.243236+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.125:0/3539121397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:16.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[50569]: debug 2023-10-09T06:07:15.831+0000 7f97bae34200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:07:16.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:15 smithi155 bash[50569]: debug 2023-10-09T06:07:15.935+0000 7f97bae34200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:07:16.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[50569]: debug 2023-10-09T06:07:16.039+0000 7f97bae34200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:07:16.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[50569]: debug 2023-10-09T06:07:16.303+0000 7f97bae34200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:17.057 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[50569]: debug 2023-10-09T06:07:16.671+0000 7f97bae34200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:07:17.057 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[50569]: debug 2023-10-09T06:07:16.835+0000 7f97bae34200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:17.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[27817]: cluster 2023-10-09T06:07:15.691192+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e27: y(active, since 1.01628s) 2023-10-09T06:07:17.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[27817]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:07:17.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[27817]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:07:17.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[27817]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:07:17.058 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:16 smithi155 bash[27817]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:07:17.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[27513]: cluster 2023-10-09T06:07:15.691192+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e27: y(active, since 1.01628s) 2023-10-09T06:07:17.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[27513]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:07:17.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[27513]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:07:17.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[27513]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:07:17.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[27513]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:07:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[33919]: cluster 2023-10-09T06:07:15.691192+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e27: y(active, since 1.01628s) 2023-10-09T06:07:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[33919]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:07:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[33919]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm [INF] [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:07:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[33919]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:07:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:16 smithi125 bash[33919]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm [INF] [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:07:17.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:17 smithi155 bash[50569]: debug 2023-10-09T06:07:17.051+0000 7f97bae34200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:07:18.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:17 smithi155 bash[27817]: cluster 2023-10-09T06:07:16.692116+0000 mgr.y (mgr.15930) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:18.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:17 smithi155 bash[27817]: cluster 2023-10-09T06:07:16.699502+0000 mon.a (mon.0) 858 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T06:07:18.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:17 smithi125 bash[27513]: cluster 2023-10-09T06:07:16.692116+0000 mgr.y (mgr.15930) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:18.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:17 smithi125 bash[27513]: cluster 2023-10-09T06:07:16.699502+0000 mon.a (mon.0) 858 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T06:07:18.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:17 smithi125 bash[33919]: cluster 2023-10-09T06:07:16.692116+0000 mgr.y (mgr.15930) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:17 smithi125 bash[33919]: cluster 2023-10-09T06:07:16.699502+0000 mon.a (mon.0) 858 : cluster [DBG] mgrmap e28: y(active, since 2s) 2023-10-09T06:07:18.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[50569]: debug 2023-10-09T06:07:18.067+0000 7f97bae34200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:07:18.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[50569]: debug 2023-10-09T06:07:18.191+0000 7f97bae34200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:07:18.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[50569]: debug 2023-10-09T06:07:18.315+0000 7f97bae34200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:07:19.053 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[27817]: audit 2023-10-09T06:07:17.700706+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.125:0/3951755621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:19.053 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[50569]: debug 2023-10-09T06:07:18.819+0000 7f97bae34200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:07:19.053 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:18 smithi155 bash[52978]: ts=2023-10-09T06:07:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:07:19.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:18 smithi125 bash[27513]: audit 2023-10-09T06:07:17.700706+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.125:0/3951755621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:19.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:18 smithi125 bash[33919]: audit 2023-10-09T06:07:17.700706+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.125:0/3951755621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:19.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:19 smithi155 bash[50569]: debug 2023-10-09T06:07:19.047+0000 7f97bae34200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:07:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:19 smithi155 bash[27817]: cluster 2023-10-09T06:07:18.692805+0000 mgr.y (mgr.15930) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:19.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:19 smithi155 bash[50569]: debug 2023-10-09T06:07:19.499+0000 7f97bae34200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:07:20.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:19 smithi125 bash[27513]: cluster 2023-10-09T06:07:18.692805+0000 mgr.y (mgr.15930) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:20.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:19 smithi125 bash[33919]: cluster 2023-10-09T06:07:18.692805+0000 mgr.y (mgr.15930) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:20.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:20 smithi155 bash[50569]: debug 2023-10-09T06:07:20.019+0000 7f97bae34200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:21.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:20 smithi125 bash[27513]: audit 2023-10-09T06:07:20.150574+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.125:0/3055469426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:21.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:20 smithi125 bash[33919]: audit 2023-10-09T06:07:20.150574+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.125:0/3055469426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:21.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:20 smithi155 bash[27817]: audit 2023-10-09T06:07:20.150574+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.125:0/3055469426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:21.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:20 smithi155 bash[50569]: debug 2023-10-09T06:07:20.671+0000 7f97bae34200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:07:21.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:20 smithi155 bash[50569]: debug 2023-10-09T06:07:20.799+0000 7f97bae34200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:07:21.517 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:21 smithi155 bash[50569]: debug 2023-10-09T06:07:21.215+0000 7f97bae34200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:07:21.517 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:21 smithi155 bash[50569]: debug 2023-10-09T06:07:21.347+0000 7f97bae34200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:07:21.517 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:21 smithi155 bash[50569]: debug 2023-10-09T06:07:21.511+0000 7f97bae34200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:21.794 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:21 smithi155 bash[50569]: debug 2023-10-09T06:07:21.647+0000 7f97bae34200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:07:22.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:21 smithi155 bash[50569]: debug 2023-10-09T06:07:21.787+0000 7f97bae34200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:07:22.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[50569]: debug 2023-10-09T06:07:22.003+0000 7f97bae34200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:07:22.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: cluster 2023-10-09T06:07:20.693279+0000 mgr.y (mgr.15930) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:22.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:21.313179+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:21.323987+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:21.367075+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:21.378184+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:22.188056+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:22.197158+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:22.229185+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[27513]: audit 2023-10-09T06:07:22.240200+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: cluster 2023-10-09T06:07:20.693279+0000 mgr.y (mgr.15930) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:21.313179+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:21.323987+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:21.367075+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:21.378184+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:22.188056+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:22.197158+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:22.229185+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:22 smithi125 bash[33919]: audit 2023-10-09T06:07:22.240200+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[50569]: debug 2023-10-09T06:07:22.143+0000 7f97bae34200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:07:22.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: cluster 2023-10-09T06:07:20.693279+0000 mgr.y (mgr.15930) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:22.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:21.313179+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:21.323987+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:21.367075+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:21.378184+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:22.188056+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:22.197158+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:22.229185+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:22 smithi155 bash[27817]: audit 2023-10-09T06:07:22.240200+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:23.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:23 smithi125 bash[27513]: audit 2023-10-09T06:07:22.627951+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.125:0/1473897055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:23.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:23 smithi125 bash[33919]: audit 2023-10-09T06:07:22.627951+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.125:0/1473897055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[27817]: audit 2023-10-09T06:07:22.627951+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.125:0/1473897055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: debug 2023-10-09T06:07:23.415+0000 7f97bae34200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: [09/Oct/2023:06:07:23] ENGINE Bus STARTING 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: CherryPy Checker: 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: The Application mounted at '' has an empty config. 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: [09/Oct/2023:06:07:23] ENGINE Serving on http://:::9283 2023-10-09T06:07:23.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:23 smithi155 bash[50569]: [09/Oct/2023:06:07:23] ENGINE Bus STARTED 2023-10-09T06:07:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: cluster 2023-10-09T06:07:22.694374+0000 mgr.y (mgr.15930) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: cluster 2023-10-09T06:07:23.440222+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: audit 2023-10-09T06:07:23.442704+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: audit 2023-10-09T06:07:23.444474+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: audit 2023-10-09T06:07:23.450366+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[27513]: audit 2023-10-09T06:07:23.451227+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: cluster 2023-10-09T06:07:22.694374+0000 mgr.y (mgr.15930) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: cluster 2023-10-09T06:07:23.440222+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: audit 2023-10-09T06:07:23.442704+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: audit 2023-10-09T06:07:23.444474+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: audit 2023-10-09T06:07:23.450366+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:24 smithi125 bash[33919]: audit 2023-10-09T06:07:23.451227+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: cluster 2023-10-09T06:07:22.694374+0000 mgr.y (mgr.15930) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: cluster 2023-10-09T06:07:23.440222+0000 mon.a (mon.0) 868 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: audit 2023-10-09T06:07:23.442704+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: audit 2023-10-09T06:07:23.444474+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: audit 2023-10-09T06:07:23.450366+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:24 smithi155 bash[27817]: audit 2023-10-09T06:07:23.451227+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.? 172.21.15.155:0/428476858' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[27513]: cluster 2023-10-09T06:07:24.339839+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T06:07:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[27513]: audit 2023-10-09T06:07:24.341287+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:07:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[27513]: audit 2023-10-09T06:07:25.084384+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.125:0/4225896668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[33919]: cluster 2023-10-09T06:07:24.339839+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T06:07:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[33919]: audit 2023-10-09T06:07:24.341287+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:07:25.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:25 smithi125 bash[33919]: audit 2023-10-09T06:07:25.084384+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.125:0/4225896668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:25 smithi155 bash[27817]: cluster 2023-10-09T06:07:24.339839+0000 mon.a (mon.0) 869 : cluster [DBG] mgrmap e29: y(active, since 9s), standbys: x 2023-10-09T06:07:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:25 smithi155 bash[27817]: audit 2023-10-09T06:07:24.341287+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:07:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:25 smithi155 bash[27817]: audit 2023-10-09T06:07:25.084384+0000 mon.a (mon.0) 870 : audit [DBG] from='client.? 172.21.15.125:0/4225896668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:26.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:26 smithi125 bash[27513]: cluster 2023-10-09T06:07:24.694970+0000 mgr.y (mgr.15930) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:26.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:26 smithi125 bash[33919]: cluster 2023-10-09T06:07:24.694970+0000 mgr.y (mgr.15930) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:26.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:26 smithi155 bash[27817]: cluster 2023-10-09T06:07:24.694970+0000 mgr.y (mgr.15930) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:27.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:27 smithi155 bash[52978]: ts=2023-10-09T06:07:27.271Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.155:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.155:8765: connect: connection refused" 2023-10-09T06:07:27.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:27 smithi155 bash[52978]: ts=2023-10-09T06:07:27.271Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.155:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.155:8765: connect: connection refused" 2023-10-09T06:07:27.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:27 smithi155 bash[52978]: ts=2023-10-09T06:07:27.274Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.155:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.155:8765: connect: connection refused" 2023-10-09T06:07:27.643 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:27 smithi155 bash[52978]: ts=2023-10-09T06:07:27.276Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.155:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.155:8765: connect: connection refused" 2023-10-09T06:07:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:28 smithi125 bash[27513]: cluster 2023-10-09T06:07:26.695922+0000 mgr.y (mgr.15930) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:28 smithi125 bash[27513]: audit 2023-10-09T06:07:27.519514+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.125:0/524483797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:28.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:28 smithi125 bash[33919]: cluster 2023-10-09T06:07:26.695922+0000 mgr.y (mgr.15930) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:28.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:28 smithi125 bash[33919]: audit 2023-10-09T06:07:27.519514+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.125:0/524483797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:28 smithi155 bash[27817]: cluster 2023-10-09T06:07:26.695922+0000 mgr.y (mgr.15930) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:28 smithi155 bash[27817]: audit 2023-10-09T06:07:27.519514+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.125:0/524483797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:29.646 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:29 smithi155 bash[50569]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:07:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[27513]: cluster 2023-10-09T06:07:28.696561+0000 mgr.y (mgr.15930) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[27513]: audit 2023-10-09T06:07:29.822576+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[27513]: audit 2023-10-09T06:07:29.971147+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.125:0/4065956451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[33919]: cluster 2023-10-09T06:07:28.696561+0000 mgr.y (mgr.15930) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[33919]: audit 2023-10-09T06:07:29.822576+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:30.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:30 smithi125 bash[33919]: audit 2023-10-09T06:07:29.971147+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.125:0/4065956451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:30 smithi155 bash[27817]: cluster 2023-10-09T06:07:28.696561+0000 mgr.y (mgr.15930) 12 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:30 smithi155 bash[27817]: audit 2023-10-09T06:07:29.822576+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:30 smithi155 bash[27817]: audit 2023-10-09T06:07:29.971147+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.125:0/4065956451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:32.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: cluster 2023-10-09T06:07:30.697260+0000 mgr.y (mgr.15930) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:32.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:31.460195+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:31.468566+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:31.471576+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:32.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:31.472675+0000 mon.c (mon.2) 186 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.143195+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.149132+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.158159+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.164729+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.176893+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.811 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:32 smithi155 bash[27817]: audit 2023-10-09T06:07:32.416666+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.125:0/1615067793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:32.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: cluster 2023-10-09T06:07:30.697260+0000 mgr.y (mgr.15930) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:32.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:31.460195+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:31.468566+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:31.471576+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:31.472675+0000 mon.c (mon.2) 186 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.143195+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.149132+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.158159+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.164729+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.176893+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[27513]: audit 2023-10-09T06:07:32.416666+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.125:0/1615067793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: cluster 2023-10-09T06:07:30.697260+0000 mgr.y (mgr.15930) 13 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:31.460195+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:31.468566+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:31.471576+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:31.472675+0000 mon.c (mon.2) 186 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.143195+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.149132+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.158159+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.164729+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.176893+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:32.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:32 smithi125 bash[33919]: audit 2023-10-09T06:07:32.416666+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.125:0/1615067793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[27513]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:07:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:33 smithi125 bash[33919]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:07:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:07:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:33 smithi155 bash[27817]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:07:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:34 smithi155 bash[27817]: cluster 2023-10-09T06:07:32.698307+0000 mgr.y (mgr.15930) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:34.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:34 smithi125 bash[27513]: cluster 2023-10-09T06:07:32.698307+0000 mgr.y (mgr.15930) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:34 smithi125 bash[33919]: cluster 2023-10-09T06:07:32.698307+0000 mgr.y (mgr.15930) 24 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:35.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:35 smithi125 bash[27513]: audit 2023-10-09T06:07:34.863597+0000 mon.a (mon.0) 879 : audit [DBG] from='client.? 172.21.15.125:0/604156911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:35 smithi125 bash[33919]: audit 2023-10-09T06:07:34.863597+0000 mon.a (mon.0) 879 : audit [DBG] from='client.? 172.21.15.125:0/604156911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:35 smithi155 bash[27817]: audit 2023-10-09T06:07:34.863597+0000 mon.a (mon.0) 879 : audit [DBG] from='client.? 172.21.15.125:0/604156911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:36.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:36 smithi155 bash[27817]: cluster 2023-10-09T06:07:34.698892+0000 mgr.y (mgr.15930) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:36.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:36 smithi125 bash[27513]: cluster 2023-10-09T06:07:34.698892+0000 mgr.y (mgr.15930) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:36.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:36 smithi125 bash[33919]: cluster 2023-10-09T06:07:34.698892+0000 mgr.y (mgr.15930) 25 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:37.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:36 smithi155 systemd[1]: Stopping Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:07:37.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:36 smithi155 bash[54830]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-09T06:07:37.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:07:37.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:07:37.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.042Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.043Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.043Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.043Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.044Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.044Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T06:07:37.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[52978]: ts=2023-10-09T06:07:37.044Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T06:07:37.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[27817]: audit 2023-10-09T06:07:37.318216+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.125:0/2394353082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:37 smithi125 bash[27513]: audit 2023-10-09T06:07:37.318216+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.125:0/2394353082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:37 smithi125 bash[33919]: audit 2023-10-09T06:07:37.318216+0000 mon.a (mon.0) 880 : audit [DBG] from='client.? 172.21.15.125:0/2394353082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:37.892 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[54864]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:07:37.892 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 bash[54927]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:07:37.892 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:07:37.892 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 systemd[1]: Stopped Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:07:38.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:37 smithi155 systemd[1]: Started Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:07:38.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:38 smithi155 bash[27817]: cluster 2023-10-09T06:07:36.699983+0000 mgr.y (mgr.15930) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:38.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:38 smithi155 bash[27817]: audit 2023-10-09T06:07:37.932195+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:38 smithi155 bash[27817]: audit 2023-10-09T06:07:37.942255+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:38 smithi155 bash[27817]: audit 2023-10-09T06:07:38.004543+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:38.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:38 smithi155 bash[27817]: audit 2023-10-09T06:07:38.013394+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[27513]: cluster 2023-10-09T06:07:36.699983+0000 mgr.y (mgr.15930) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[27513]: audit 2023-10-09T06:07:37.932195+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[27513]: audit 2023-10-09T06:07:37.942255+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[27513]: audit 2023-10-09T06:07:38.004543+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[27513]: audit 2023-10-09T06:07:38.013394+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[33919]: cluster 2023-10-09T06:07:36.699983+0000 mgr.y (mgr.15930) 26 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[33919]: audit 2023-10-09T06:07:37.932195+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[33919]: audit 2023-10-09T06:07:37.942255+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[33919]: audit 2023-10-09T06:07:38.004543+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:38 smithi125 bash[33919]: audit 2023-10-09T06:07:38.013394+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:39.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[27817]: audit 2023-10-09T06:07:38.005837+0000 mgr.y (mgr.15930) 27 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.609Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.609Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.609Z caller=main.go:518 level=info host_details="(Linux 5.15.0-86-generic #96~20.04.1-Ubuntu SMP Thu Sep 21 13:23:37 UTC 2023 x86_64 smithi155 (none))" 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.609Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.609Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.611Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.612Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.615Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-09T06:07:39.791 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.615Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T06:07:39.792 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.616Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=796.668µs 2023-10-09T06:07:39.792 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.616Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T06:07:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:39 smithi125 bash[27513]: audit 2023-10-09T06:07:38.005837+0000 mgr.y (mgr.15930) 27 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:39 smithi125 bash[33919]: audit 2023-10-09T06:07:38.005837+0000 mgr.y (mgr.15930) 27 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:07:40.051 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:39 smithi155 bash[54956]: ts=2023-10-09T06:07:39.789Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T06:07:40.051 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.000Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.050Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.050Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.050Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=92.715µs wal_replay_duration=433.951875ms total_replay_duration=434.865445ms 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.057Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.057Z caller=main.go:947 level=info msg="TSDB started" 2023-10-09T06:07:40.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.057Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T06:07:40.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.083Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.053164ms db_storage=1.412µs remote_storage=2.498µs web_handler=716ns query_engine=1.309µs scrape=449.187µs scrape_sd=166.348µs notify=26.952µs notify_sd=19.717µs rules=24.777528ms 2023-10-09T06:07:40.394 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[54956]: ts=2023-10-09T06:07:40.083Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-09T06:07:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:40 smithi125 bash[33919]: cluster 2023-10-09T06:07:38.700582+0000 mgr.y (mgr.15930) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:40.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:40 smithi125 bash[33919]: audit 2023-10-09T06:07:39.767598+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.125:0/2646366952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:40 smithi125 bash[27513]: cluster 2023-10-09T06:07:38.700582+0000 mgr.y (mgr.15930) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:40.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:40 smithi125 bash[27513]: audit 2023-10-09T06:07:39.767598+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.125:0/2646366952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[27817]: cluster 2023-10-09T06:07:38.700582+0000 mgr.y (mgr.15930) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:40 smithi155 bash[27817]: audit 2023-10-09T06:07:39.767598+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.125:0/2646366952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:41.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:41 smithi155 bash[27817]: audit 2023-10-09T06:07:40.589553+0000 mon.c (mon.2) 191 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:41 smithi155 bash[27817]: audit 2023-10-09T06:07:40.590277+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:41 smithi155 bash[27817]: audit 2023-10-09T06:07:40.592402+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:41.809 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:41 smithi155 bash[27817]: audit 2023-10-09T06:07:40.594111+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[27513]: audit 2023-10-09T06:07:40.589553+0000 mon.c (mon.2) 191 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[27513]: audit 2023-10-09T06:07:40.590277+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[27513]: audit 2023-10-09T06:07:40.592402+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:41.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[27513]: audit 2023-10-09T06:07:40.594111+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[33919]: audit 2023-10-09T06:07:40.589553+0000 mon.c (mon.2) 191 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[33919]: audit 2023-10-09T06:07:40.590277+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[33919]: audit 2023-10-09T06:07:40.592402+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:41 smithi125 bash[33919]: audit 2023-10-09T06:07:40.594111+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[27513]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:07:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[27513]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:07:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[27513]: cluster 2023-10-09T06:07:40.701213+0000 mgr.y (mgr.15930) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:42.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[27513]: audit 2023-10-09T06:07:42.217569+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.125:0/3511157775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[33919]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:07:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[33919]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:07:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[33919]: cluster 2023-10-09T06:07:40.701213+0000 mgr.y (mgr.15930) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:42.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:42 smithi125 bash[33919]: audit 2023-10-09T06:07:42.217569+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.125:0/3511157775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:42.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:42 smithi155 bash[27817]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-09T06:07:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:42 smithi155 bash[27817]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm [INF] Deploying daemon mgr.x on smithi155 2023-10-09T06:07:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:42 smithi155 bash[27817]: cluster 2023-10-09T06:07:40.701213+0000 mgr.y (mgr.15930) 31 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:42 smithi155 bash[27817]: audit 2023-10-09T06:07:42.217569+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.125:0/3511157775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:43.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 systemd[1]: Stopping Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:07:44.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 bash[55268]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T06:07:44.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:07:44.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 bash[55331]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T06:07:44.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T06:07:44.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:43 smithi155 systemd[1]: Stopped Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:07:44.525 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:44 smithi155 systemd[1]: Started Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:07:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[33919]: cluster 2023-10-09T06:07:42.702173+0000 mgr.y (mgr.15930) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[33919]: audit 2023-10-09T06:07:44.249278+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[33919]: audit 2023-10-09T06:07:44.259142+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[33919]: audit 2023-10-09T06:07:44.269857+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:44.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[27513]: cluster 2023-10-09T06:07:42.702173+0000 mgr.y (mgr.15930) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:44.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[27513]: audit 2023-10-09T06:07:44.249278+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[27513]: audit 2023-10-09T06:07:44.259142+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:44 smithi125 bash[27513]: audit 2023-10-09T06:07:44.269857+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:44 smithi155 bash[27817]: cluster 2023-10-09T06:07:42.702173+0000 mgr.y (mgr.15930) 32 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:44 smithi155 bash[27817]: audit 2023-10-09T06:07:44.249278+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:44 smithi155 bash[27817]: audit 2023-10-09T06:07:44.259142+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:44 smithi155 bash[27817]: audit 2023-10-09T06:07:44.269857+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:45.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:45 smithi155 bash[27817]: audit 2023-10-09T06:07:44.665216+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.125:0/30289573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:45.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:45 smithi155 bash[27817]: audit 2023-10-09T06:07:44.823177+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:45.617 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:45 smithi155 bash[55402]: debug 2023-10-09T06:07:45.438+0000 7f4619423700 1 -- 172.21.15.155:0/1340354439 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556eeea8d1e0 con 0x556eef80c400 2023-10-09T06:07:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:45 smithi125 bash[27513]: audit 2023-10-09T06:07:44.665216+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.125:0/30289573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:45 smithi125 bash[27513]: audit 2023-10-09T06:07:44.823177+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:45 smithi125 bash[33919]: audit 2023-10-09T06:07:44.665216+0000 mon.a (mon.0) 886 : audit [DBG] from='client.? 172.21.15.125:0/30289573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:45 smithi125 bash[33919]: audit 2023-10-09T06:07:44.823177+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:45.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:45 smithi155 bash[55402]: debug 2023-10-09T06:07:45.606+0000 7f4623571200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:07:46.797 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:46 smithi155 bash[27817]: cluster 2023-10-09T06:07:44.702843+0000 mgr.y (mgr.15930) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:46.797 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:46 smithi155 bash[55402]: debug 2023-10-09T06:07:46.566+0000 7f4623571200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:07:46.797 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:46 smithi155 bash[55402]: debug 2023-10-09T06:07:46.678+0000 7f4623571200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:07:46.798 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:46 smithi155 bash[55402]: debug 2023-10-09T06:07:46.790+0000 7f4623571200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:07:46.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:46 smithi125 bash[27513]: cluster 2023-10-09T06:07:44.702843+0000 mgr.y (mgr.15930) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:46 smithi125 bash[33919]: cluster 2023-10-09T06:07:44.702843+0000 mgr.y (mgr.15930) 33 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:47.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[55402]: debug 2023-10-09T06:07:47.062+0000 7f4623571200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:47.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[27817]: cluster 2023-10-09T06:07:46.703621+0000 mgr.y (mgr.15930) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:47.825 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[27817]: audit 2023-10-09T06:07:47.111788+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.125:0/3720796924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:47.825 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[55402]: debug 2023-10-09T06:07:47.426+0000 7f4623571200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:07:47.825 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[55402]: debug 2023-10-09T06:07:47.594+0000 7f4623571200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:47 smithi125 bash[27513]: cluster 2023-10-09T06:07:46.703621+0000 mgr.y (mgr.15930) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:47 smithi125 bash[27513]: audit 2023-10-09T06:07:47.111788+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.125:0/3720796924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:47 smithi125 bash[33919]: cluster 2023-10-09T06:07:46.703621+0000 mgr.y (mgr.15930) 34 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:47 smithi125 bash[33919]: audit 2023-10-09T06:07:47.111788+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.125:0/3720796924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:48.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:47 smithi155 bash[55402]: debug 2023-10-09T06:07:47.818+0000 7f4623571200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:07:49.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:48 smithi155 bash[55402]: debug 2023-10-09T06:07:48.858+0000 7f4623571200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:07:49.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:48 smithi155 bash[55402]: debug 2023-10-09T06:07:48.978+0000 7f4623571200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:07:49.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:49 smithi155 bash[55402]: debug 2023-10-09T06:07:49.098+0000 7f4623571200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:07:49.906 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:49 smithi155 bash[55402]: debug 2023-10-09T06:07:49.642+0000 7f4623571200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:07:49.906 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:49 smithi155 bash[27817]: cluster 2023-10-09T06:07:48.704242+0000 mgr.y (mgr.15930) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:49.906 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:49 smithi155 bash[27817]: audit 2023-10-09T06:07:49.556514+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.125:0/732514766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:50.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:49 smithi125 bash[27513]: cluster 2023-10-09T06:07:48.704242+0000 mgr.y (mgr.15930) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:50.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:49 smithi125 bash[27513]: audit 2023-10-09T06:07:49.556514+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.125:0/732514766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:50.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:49 smithi125 bash[33919]: cluster 2023-10-09T06:07:48.704242+0000 mgr.y (mgr.15930) 35 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:50.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:49 smithi125 bash[33919]: audit 2023-10-09T06:07:49.556514+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.125:0/732514766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:50.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:49 smithi155 bash[55402]: debug 2023-10-09T06:07:49.898+0000 7f4623571200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:07:50.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:50 smithi155 bash[55402]: debug 2023-10-09T06:07:50.478+0000 7f4623571200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:07:51.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[55402]: debug 2023-10-09T06:07:51.042+0000 7f4623571200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:07:51.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:07:51] "GET /metrics HTTP/1.1" 200 33931 "" "Prometheus/2.33.4" 2023-10-09T06:07:51.809 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[55402]: debug 2023-10-09T06:07:51.678+0000 7f4623571200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:07:51.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[27817]: audit 2023-10-09T06:07:50.471233+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[27817]: audit 2023-10-09T06:07:50.488396+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[27817]: audit 2023-10-09T06:07:51.339377+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.810 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[27817]: audit 2023-10-09T06:07:51.349158+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[33919]: audit 2023-10-09T06:07:50.471233+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[33919]: audit 2023-10-09T06:07:50.488396+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[33919]: audit 2023-10-09T06:07:51.339377+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[33919]: audit 2023-10-09T06:07:51.349158+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[27513]: audit 2023-10-09T06:07:50.471233+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[27513]: audit 2023-10-09T06:07:50.488396+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[27513]: audit 2023-10-09T06:07:51.339377+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:51 smithi125 bash[27513]: audit 2023-10-09T06:07:51.349158+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:07:52.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:51 smithi155 bash[55402]: debug 2023-10-09T06:07:51.802+0000 7f4623571200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:07:52.464 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.190+0000 7f4623571200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:07:52.464 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.314+0000 7f4623571200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:07:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[27817]: cluster 2023-10-09T06:07:50.704940+0000 mgr.y (mgr.15930) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[27817]: audit 2023-10-09T06:07:52.004194+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.125:0/2209271323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:52.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.458+0000 7f4623571200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:07:52.720 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.578+0000 7f4623571200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:07:52.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:52 smithi125 bash[27513]: cluster 2023-10-09T06:07:50.704940+0000 mgr.y (mgr.15930) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:52 smithi125 bash[27513]: audit 2023-10-09T06:07:52.004194+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.125:0/2209271323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:52.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:52 smithi125 bash[33919]: cluster 2023-10-09T06:07:50.704940+0000 mgr.y (mgr.15930) 36 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:52 smithi125 bash[33919]: audit 2023-10-09T06:07:52.004194+0000 mon.a (mon.0) 892 : audit [DBG] from='client.? 172.21.15.125:0/2209271323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:53.042 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.714+0000 7f4623571200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:07:53.042 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:52 smithi155 bash[55402]: debug 2023-10-09T06:07:52.910+0000 7f4623571200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:07:53.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:53 smithi155 bash[55402]: debug 2023-10-09T06:07:53.034+0000 7f4623571200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:07:54.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: debug 2023-10-09T06:07:54.274+0000 7f4623571200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:07:54.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: [09/Oct/2023:06:07:54] ENGINE Bus STARTING 2023-10-09T06:07:54.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: CherryPy Checker: 2023-10-09T06:07:54.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: The Application mounted at '' has an empty config. 2023-10-09T06:07:54.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: [09/Oct/2023:06:07:54] ENGINE Serving on http://:::9283 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[55402]: [09/Oct/2023:06:07:54] ENGINE Bus STARTED 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: cluster 2023-10-09T06:07:52.705884+0000 mgr.y (mgr.15930) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: cluster 2023-10-09T06:07:54.300629+0000 mon.a (mon.0) 893 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: cluster 2023-10-09T06:07:54.300854+0000 mon.a (mon.0) 894 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: audit 2023-10-09T06:07:54.307868+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: audit 2023-10-09T06:07:54.309440+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: audit 2023-10-09T06:07:54.314621+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: audit 2023-10-09T06:07:54.315487+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:54 smithi155 bash[27817]: audit 2023-10-09T06:07:54.447332+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.125:0/3098351868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:54.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: cluster 2023-10-09T06:07:52.705884+0000 mgr.y (mgr.15930) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: cluster 2023-10-09T06:07:54.300629+0000 mon.a (mon.0) 893 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T06:07:54.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: cluster 2023-10-09T06:07:54.300854+0000 mon.a (mon.0) 894 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:54.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: audit 2023-10-09T06:07:54.307868+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: audit 2023-10-09T06:07:54.309440+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: audit 2023-10-09T06:07:54.314621+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: audit 2023-10-09T06:07:54.315487+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[27513]: audit 2023-10-09T06:07:54.447332+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.125:0/3098351868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: cluster 2023-10-09T06:07:52.705884+0000 mgr.y (mgr.15930) 37 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: cluster 2023-10-09T06:07:54.300629+0000 mon.a (mon.0) 893 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: cluster 2023-10-09T06:07:54.300854+0000 mon.a (mon.0) 894 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: audit 2023-10-09T06:07:54.307868+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:07:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: audit 2023-10-09T06:07:54.309440+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:07:54.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: audit 2023-10-09T06:07:54.314621+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:07:54.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: audit 2023-10-09T06:07:54.315487+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.? 172.21.15.155:0/3107195715' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:07:54.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:54 smithi125 bash[33919]: audit 2023-10-09T06:07:54.447332+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.125:0/3098351868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:55.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:55 smithi125 bash[27513]: cluster 2023-10-09T06:07:54.508772+0000 mon.a (mon.0) 896 : cluster [DBG] mgrmap e30: y(active, since 39s), standbys: x 2023-10-09T06:07:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:55 smithi125 bash[33919]: cluster 2023-10-09T06:07:54.508772+0000 mon.a (mon.0) 896 : cluster [DBG] mgrmap e30: y(active, since 39s), standbys: x 2023-10-09T06:07:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:55 smithi155 bash[27817]: cluster 2023-10-09T06:07:54.508772+0000 mon.a (mon.0) 896 : cluster [DBG] mgrmap e30: y(active, since 39s), standbys: x 2023-10-09T06:07:56.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:56 smithi125 bash[33919]: cluster 2023-10-09T06:07:54.706567+0000 mgr.y (mgr.15930) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:56.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:56 smithi125 bash[27513]: cluster 2023-10-09T06:07:54.706567+0000 mgr.y (mgr.15930) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:56.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:56 smithi155 bash[27817]: cluster 2023-10-09T06:07:54.706567+0000 mgr.y (mgr.15930) 38 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:57 smithi155 bash[27817]: audit 2023-10-09T06:07:56.897996+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.125:0/3634744104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:57.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:57 smithi125 bash[27513]: audit 2023-10-09T06:07:56.897996+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.125:0/3634744104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:57.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:57 smithi125 bash[33919]: audit 2023-10-09T06:07:56.897996+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.125:0/3634744104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:58.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:58 smithi155 bash[27817]: cluster 2023-10-09T06:07:56.707608+0000 mgr.y (mgr.15930) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:58.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:58 smithi125 bash[27513]: cluster 2023-10-09T06:07:56.707608+0000 mgr.y (mgr.15930) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:58.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:58 smithi125 bash[33919]: cluster 2023-10-09T06:07:56.707608+0000 mgr.y (mgr.15930) 39 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:59.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:07:58 smithi155 bash[54956]: ts=2023-10-09T06:07:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:07:59.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:59 smithi125 bash[33919]: cluster 2023-10-09T06:07:58.708240+0000 mgr.y (mgr.15930) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:59.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:07:59 smithi125 bash[33919]: audit 2023-10-09T06:07:59.342419+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.125:0/2061882571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:59.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:59 smithi125 bash[27513]: cluster 2023-10-09T06:07:58.708240+0000 mgr.y (mgr.15930) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:59.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:07:59 smithi125 bash[27513]: audit 2023-10-09T06:07:59.342419+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.125:0/2061882571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:07:59.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:59 smithi155 bash[27817]: cluster 2023-10-09T06:07:58.708240+0000 mgr.y (mgr.15930) 40 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:07:59 smithi155 bash[27817]: audit 2023-10-09T06:07:59.342419+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.125:0/2061882571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:00.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:00 smithi125 bash[33919]: audit 2023-10-09T06:07:59.823967+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:00 smithi125 bash[27513]: audit 2023-10-09T06:07:59.823967+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:00.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:00 smithi155 bash[27817]: audit 2023-10-09T06:07:59.823967+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:01.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:08:01] "GET /metrics HTTP/1.1" 200 33928 "" "Prometheus/2.33.4" 2023-10-09T06:08:02.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: cluster 2023-10-09T06:08:00.708901+0000 mgr.y (mgr.15930) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.852698+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.862443+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.864560+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.865885+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.873330+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.957987+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.964653+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.974114+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.976045+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.976597+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.981659+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.983802+0000 mon.c (mon.2) 203 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.984247+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.989750+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.991895+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:00.993064+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T06:08:02.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[33919]: audit 2023-10-09T06:08:01.779417+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.125:0/3722757007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: cluster 2023-10-09T06:08:00.708901+0000 mgr.y (mgr.15930) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.852698+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.862443+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.864560+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.865885+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.873330+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.957987+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.964653+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.974114+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.976045+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.976597+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.981659+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.983802+0000 mon.c (mon.2) 203 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.984247+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.989750+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.991895+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:00.993064+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T06:08:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:01 smithi125 bash[27513]: audit 2023-10-09T06:08:01.779417+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.125:0/3722757007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: cluster 2023-10-09T06:08:00.708901+0000 mgr.y (mgr.15930) 41 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.852698+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.862443+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.864560+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.865885+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.873330+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.957987+0000 mon.c (mon.2) 200 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.964653+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.974114+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.976045+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.976597+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.981659+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.983802+0000 mon.c (mon.2) 203 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.984247+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.989750+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.15930 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:08:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.991895+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:08:02.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:00.993064+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:08:02.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-09T06:08:02.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:01 smithi155 bash[27817]: audit 2023-10-09T06:08:01.779417+0000 mon.c (mon.2) 206 : audit [DBG] from='client.? 172.21.15.125:0/3722757007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:04.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: cluster 2023-10-09T06:08:02.709501+0000 mgr.y (mgr.15930) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:04.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T06:08:04.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: audit 2023-10-09T06:08:03.472012+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: audit 2023-10-09T06:08:03.474051+0000 mon.c (mon.2) 207 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: audit 2023-10-09T06:08:03.475229+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: audit 2023-10-09T06:08:03.476463+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm [INF] Deploying daemon mon.a on smithi125 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[27513]: audit 2023-10-09T06:08:04.206466+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.125:0/390506244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:04.797 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: cluster 2023-10-09T06:08:02.709501+0000 mgr.y (mgr.15930) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: audit 2023-10-09T06:08:03.472012+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: audit 2023-10-09T06:08:03.474051+0000 mon.c (mon.2) 207 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: audit 2023-10-09T06:08:03.475229+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: audit 2023-10-09T06:08:03.476463+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:04.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm [INF] Deploying daemon mon.a on smithi125 2023-10-09T06:08:04.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:04 smithi125 bash[33919]: audit 2023-10-09T06:08:04.206466+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.125:0/390506244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: cluster 2023-10-09T06:08:02.709501+0000 mgr.y (mgr.15930) 44 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm [INF] Upgrade: Updating mon.a 2023-10-09T06:08:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: audit 2023-10-09T06:08:03.472012+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15930 ' entity='mgr.y' 2023-10-09T06:08:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: audit 2023-10-09T06:08:03.474051+0000 mon.c (mon.2) 207 : audit [INF] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:08:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: audit 2023-10-09T06:08:03.475229+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:08:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: audit 2023-10-09T06:08:03.476463+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.15930 172.21.15.125:0/823052127' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm [INF] Deploying daemon mon.a on smithi125 2023-10-09T06:08:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:04 smithi155 bash[27817]: audit 2023-10-09T06:08:04.206466+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.125:0/390506244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:06.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:05 smithi125 systemd[1]: Stopping Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:08:06.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 bash[27513]: debug 2023-10-09T06:08:06.050+0000 7f79dd5a0700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:08:06.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 bash[27513]: debug 2023-10-09T06:08:06.050+0000 7f79dd5a0700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T06:08:07.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 bash[77157]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T06:08:07.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 bash[77244]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T06:08:07.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T06:08:07.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:06 smithi125 systemd[1]: Stopped Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:08:07.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:07 smithi125 systemd[1]: Started Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.290+0000 7ff9587c3c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.290+0000 7ff9587c3c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.290+0000 7ff9587c3c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 0 load: jerasure load: lrc 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Git sha 0 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: DB SUMMARY 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: DB Session ID: 46TEUC0VJG1ZB3E5UFZQ 2023-10-09T06:08:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 966 Bytes 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 15867174 ; 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:08:08.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.env: 0x55b7c8149720 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.info_log: 0x55b7cb15d480 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T06:08:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T06:08:08.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:08:08.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.db_log_dir: 2023-10-09T06:08:08.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.wal_dir: 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.write_buffer_manager: 0x55b7ca3eac80 2023-10-09T06:08:08.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:08:08.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.row_cache: None 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.wal_filter: None 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:08:08.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T06:08:08.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T06:08:08.646 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Compression algorithms supported: 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kZSTD supported: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T06:08:08.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.merge_operator: 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:08:08.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b7cb15d580) 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: cache_index_and_filter_blocks: 1 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: pin_top_level_index_and_filter: 1 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: index_type: 0 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: data_block_index_type: 0 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: index_shortening: 1 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: checksum: 4 2023-10-09T06:08:08.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: no_block_cache: 0 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_cache: 0x55b7ca46d1f0 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_cache_name: BinnedLRUCache 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_cache_options: 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: capacity : 536870912 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: num_shard_bits : 4 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: strict_capacity_limit : 0 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: high_pri_pool_ratio: 0.000 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_cache_compressed: (nil) 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: persistent_cache: (nil) 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_size: 4096 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_size_deviation: 10 2023-10-09T06:08:08.650 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_restart_interval: 16 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: index_block_restart_interval: 1 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: metadata_block_size: 4096 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: partition_filters: 0 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: use_delta_encoding: 1 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: filter_policy: bloomfilter 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: whole_key_filtering: 1 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: verify_compression: 0 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: read_amp_bytes_per_bit: 0 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: format_version: 5 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: enable_index_compression: 1 2023-10-09T06:08:08.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: block_align: 0 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: max_auto_readahead_size: 262144 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: prepopulate_block_cache: 0 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: initial_auto_readahead_size: 8192 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: num_file_reads_for_auto_readahead: 2 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:08:08.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.num_levels: 7 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:08:08.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:08:08.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:08:08.655 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:08:08.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:08:08.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:08:08.658 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:08:08.659 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff9587c3c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.294+0000 7ff94dfe1700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11343, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 973ebfba-e512-4302-a736-e20ba562f485 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831688299378, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.298+0000 7ff9587c3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.353+0000 7ff9587c3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831688358251, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 15364129, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11344, "largest_seqno": 15199, "table_properties": {"data_size": 15341982, "index_size": 17605, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3525, "raw_key_size": 26853, "raw_average_key_size": 19, "raw_value_size": 15304987, "raw_average_value_size": 11050, "num_data_blocks": 765, "num_entries": 1385, "num_filter_entries": 1385, "num_deletions": 259, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831688, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "973ebfba-e512-4302-a736-e20ba562f485", "db_session_id": "46TEUC0VJG1ZB3E5UFZQ", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.353+0000 7ff9587c3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831688358369, "job": 1, "event": "recovery_finished"} 2023-10-09T06:08:08.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.353+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-09T06:08:08.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff9587c3c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:08:08.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff9587c3c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:08:08.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff9587c3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b7ca50e000 2023-10-09T06:08:08.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff9587c3c80 4 rocksdb: DB pointer 0x55b7ca4fc000 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff946fd3700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff946fd3700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: ** DB Stats ** 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:08:08.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: ** Compaction Stats [default] ** 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: L0 1/0 14.65 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.9 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: L6 1/0 22.10 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Sum 2/0 36.75 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.9 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.9 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: ** Compaction Stats [default] ** 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:08:08.663 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 529.9 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Flush(GB): cumulative 0.014, interval 0.014 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Cumulative compaction: 0.01 GB write, 230.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Interval compaction: 0.01 GB write, 230.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:08:08.664 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Block cache BinnedLRUCache@0x55b7ca46d1f0#7 capacity: 512.00 MB usage: 24.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.5e-05 secs_since: 0 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.357+0000 7ff9587c3c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] at bind addrs [v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 1 mon.a@-1(???) e3 preinit fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).mds e7 new map 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).mds e7 print_map 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: e7 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T06:08:08.665 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: legacy client fscid: 1 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Filesystem 'foo' (1) 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: fs_name foo 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: epoch 5 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: created 2023-10-09T06:01:50.652169+0000 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: modified 2023-10-09T06:01:58.450626+0000 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: tableserver 0 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: root 0 2023-10-09T06:08:08.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: session_timeout 60 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: session_autoclose 300 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: max_file_size 1099511627776 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: required_client_features {} 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: last_failure 0 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: last_failure_osd_epoch 0 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: max_mds 1 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: in 0 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: up {0=24542} 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: failed 2023-10-09T06:08:08.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: damaged 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: stopped 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: data_pools [4] 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: metadata_pool 3 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: inline_data disabled 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: balancer 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: bal_rank_mask -1 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: standby_count_wanted 1 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: [mds.foo.smithi125.yetyhy{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: 2023-10-09T06:08:08.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: Standby daemons: 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: [mds.foo.smithi155.itxbmb{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 0 mon.a@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.361+0000 7ff9587c3c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T06:08:08.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T06:08:08.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:08:08 smithi125 bash[77299]: debug 2023-10-09T06:08:08.365+0000 7ff9587c3c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T06:08:09.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:08:08 smithi155 bash[54956]: ts=2023-10-09T06:08:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:08:10.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[73481]: ignoring --setuser ceph since I am not root 2023-10-09T06:08:10.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[73481]: ignoring --setgroup ceph since I am not root 2023-10-09T06:08:10.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[73481]: debug 2023-10-09T06:08:09.921+0000 7f84d56ae700 1 -- 172.21.15.125:0/2709025928 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5649123c91e0 con 0x564913148800 2023-10-09T06:08:10.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:10 smithi125 bash[73481]: debug 2023-10-09T06:08:10.077+0000 7f84df3de200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: audit 2023-10-09T06:08:06.656341+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.125:0/364179830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:06.710752+0000 mgr.y (mgr.15930) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:08.711313+0000 mgr.y (mgr.15930) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: audit 2023-10-09T06:08:09.107738+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.125:0/3643839579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.784508+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.807322+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.819419+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:08:10.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.819995+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:08:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.820037+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:08:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.820998+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 55s), standbys: x 2023-10-09T06:08:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.850312+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 55s), standbys: x 2023-10-09T06:08:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: cluster 2023-10-09T06:08:09.850371+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T06:08:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:09 smithi125 bash[33919]: audit 2023-10-09T06:08:09.850515+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15930 ' entity='' 2023-10-09T06:08:10.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[55402]: ignoring --setuser ceph since I am not root 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[55402]: ignoring --setgroup ceph since I am not root 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[55402]: debug 2023-10-09T06:08:09.910+0000 7fba9e5c0700 1 -- 172.21.15.155:0/526825622 <== mon.1 v2:172.21.15.155:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560f050c51e0 con 0x560f05e44c00 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:10 smithi155 bash[55402]: debug 2023-10-09T06:08:10.070+0000 7fbaa82f0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: audit 2023-10-09T06:08:06.656341+0000 mon.c (mon.2) 211 : audit [DBG] from='client.? 172.21.15.125:0/364179830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:06.710752+0000 mgr.y (mgr.15930) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:08.711313+0000 mgr.y (mgr.15930) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: audit 2023-10-09T06:08:09.107738+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.125:0/3643839579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.784508+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.807322+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.819419+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.819995+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.820037+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.820998+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e30: y(active, since 55s), standbys: x 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.850312+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e31: y(active, since 55s), standbys: x 2023-10-09T06:08:10.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: cluster 2023-10-09T06:08:09.850371+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-09T06:08:10.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:09 smithi155 bash[27817]: audit 2023-10-09T06:08:09.850515+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15930 ' entity='' 2023-10-09T06:08:11.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[73481]: debug 2023-10-09T06:08:11.021+0000 7f84df3de200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:08:11.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[73481]: debug 2023-10-09T06:08:11.121+0000 7f84df3de200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:08:11.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[73481]: debug 2023-10-09T06:08:11.225+0000 7f84df3de200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:08:11.392 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[55402]: debug 2023-10-09T06:08:11.014+0000 7fbaa82f0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:08:11.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[55402]: debug 2023-10-09T06:08:11.114+0000 7fbaa82f0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:08:11.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[55402]: debug 2023-10-09T06:08:11.214+0000 7fbaa82f0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:08:11.684 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T06:08:11.853 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[55402]: debug 2023-10-09T06:08:11.486+0000 7fbaa82f0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:08:11.853 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[27817]: audit 2023-10-09T06:08:11.667622+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.125:0/78063177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:11.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[73481]: debug 2023-10-09T06:08:11.569+0000 7f84df3de200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:08:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[33919]: audit 2023-10-09T06:08:11.667622+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.125:0/78063177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:08:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:11 smithi155 bash[55402]: debug 2023-10-09T06:08:11.846+0000 7fbaa82f0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:08:12.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:12 smithi155 bash[55402]: debug 2023-10-09T06:08:12.006+0000 7fbaa82f0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:08:12.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:11 smithi125 bash[73481]: debug 2023-10-09T06:08:11.937+0000 7f84df3de200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:08:12.322 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:12 smithi125 bash[73481]: debug 2023-10-09T06:08:12.097+0000 7f84df3de200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:08:12.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:08:12.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:12 smithi125 bash[73481]: debug 2023-10-09T06:08:12.317+0000 7f84df3de200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:08:12.642 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:12 smithi155 bash[55402]: debug 2023-10-09T06:08:12.226+0000 7fbaa82f0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:08:13.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:13 smithi125 bash[73481]: debug 2023-10-09T06:08:13.345+0000 7f84df3de200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:08:13.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:13 smithi125 bash[73481]: debug 2023-10-09T06:08:13.461+0000 7f84df3de200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:08:13.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:13 smithi125 bash[73481]: debug 2023-10-09T06:08:13.585+0000 7f84df3de200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:08:13.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:08:13 smithi125 bash[72578]: level=warn ts=2023-10-09T06:08:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi125.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.125:8443: connect: connection refused" 2023-10-09T06:08:13.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:08:13 smithi125 bash[72578]: level=warn ts=2023-10-09T06:08:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi155.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.155:8443: connect: connection refused" 2023-10-09T06:08:13.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:13 smithi155 bash[55402]: debug 2023-10-09T06:08:13.242+0000 7fbaa82f0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:08:13.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:13 smithi155 bash[55402]: debug 2023-10-09T06:08:13.358+0000 7fbaa82f0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:08:13.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:13 smithi155 bash[55402]: debug 2023-10-09T06:08:13.478+0000 7fbaa82f0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:08:14.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:14 smithi125 bash[73481]: debug 2023-10-09T06:08:14.077+0000 7f84df3de200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:08:14.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:14 smithi125 bash[73481]: debug 2023-10-09T06:08:14.305+0000 7f84df3de200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:08:14.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:13 smithi155 bash[55402]: debug 2023-10-09T06:08:13.966+0000 7fbaa82f0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:08:14.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:14 smithi155 bash[55402]: debug 2023-10-09T06:08:14.194+0000 7fbaa82f0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:08:15.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:14 smithi125 bash[73481]: debug 2023-10-09T06:08:14.757+0000 7f84df3de200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:08:15.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:14 smithi155 bash[55402]: debug 2023-10-09T06:08:14.694+0000 7fbaa82f0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:08:15.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:15 smithi125 bash[73481]: debug 2023-10-09T06:08:15.249+0000 7f84df3de200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:08:15.643 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:15 smithi155 bash[55402]: debug 2023-10-09T06:08:15.198+0000 7fbaa82f0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:08:16.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:15 smithi125 bash[73481]: debug 2023-10-09T06:08:15.813+0000 7f84df3de200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:08:16.136 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:15 smithi125 bash[73481]: debug 2023-10-09T06:08:15.933+0000 7f84df3de200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:08:16.142 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:15 smithi155 bash[55402]: debug 2023-10-09T06:08:15.754+0000 7fbaa82f0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:08:16.143 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:15 smithi155 bash[55402]: debug 2023-10-09T06:08:15.870+0000 7fbaa82f0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:08:16.489 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.218+0000 7fbaa82f0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:08:16.489 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.338+0000 7fbaa82f0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:08:16.557 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.293+0000 7f84df3de200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:08:16.558 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.405+0000 7f84df3de200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:08:16.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.553+0000 7f84df3de200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:08:16.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.669+0000 7f84df3de200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:08:16.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.797+0000 7f84df3de200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:08:16.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.482+0000 7fbaa82f0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:08:16.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.598+0000 7fbaa82f0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:08:16.893 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.726+0000 7fbaa82f0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:08:17.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:16 smithi125 bash[73481]: debug 2023-10-09T06:08:16.993+0000 7f84df3de200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:08:17.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:17 smithi125 bash[73481]: debug 2023-10-09T06:08:17.121+0000 7f84df3de200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:08:17.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:16 smithi155 bash[55402]: debug 2023-10-09T06:08:16.918+0000 7fbaa82f0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:08:17.393 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:17 smithi155 bash[55402]: debug 2023-10-09T06:08:17.042+0000 7fbaa82f0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.314768+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.314889+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: audit 2023-10-09T06:08:18.320208+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: audit 2023-10-09T06:08:18.321864+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: audit 2023-10-09T06:08:18.326099+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: audit 2023-10-09T06:08:18.328089+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[33919]: debug 2023-10-09T06:08:18.405+0000 7f97d020c700 -1 mon.c@2(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:08:18.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.377+0000 7f84df3de200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:08:18.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: [09/Oct/2023:06:08:18] ENGINE Bus STARTING 2023-10-09T06:08:18.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.314768+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.314889+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: audit 2023-10-09T06:08:18.320208+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: audit 2023-10-09T06:08:18.321864+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: audit 2023-10-09T06:08:18.326099+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: audit 2023-10-09T06:08:18.328089+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.155:0/2879127344' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[27817]: debug 2023-10-09T06:08:18.406+0000 7f38995fe700 -1 mon.b@1(peon).osd e84 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: debug 2023-10-09T06:08:18.306+0000 7fbaa82f0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: [09/Oct/2023:06:08:18] ENGINE Bus STARTING 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: CherryPy Checker: 2023-10-09T06:08:18.639 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: The Application mounted at '' has an empty config. 2023-10-09T06:08:18.640 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: [09/Oct/2023:06:08:18] ENGINE Serving on http://:::9283 2023-10-09T06:08:18.640 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:08:18 smithi155 bash[55402]: [09/Oct/2023:06:08:18] ENGINE Bus STARTED 2023-10-09T06:08:18.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.637+0000 7f8439f73700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.637+0000 7f8439f73700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.637+0000 7f8439f73700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.637+0000 7f8439f73700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.637+0000 7f8439f73700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.641+0000 7f843ff7f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.641+0000 7f843ff7f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.641+0000 7f843ff7f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.641+0000 7f843ff7f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: debug 2023-10-09T06:08:18.641+0000 7f843ff7f700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: CherryPy Checker: 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: The Application mounted at '' has an empty config. 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: [09/Oct/2023:06:08:18] ENGINE Serving on http://:::9283 2023-10-09T06:08:18.887 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:18 smithi125 bash[73481]: [09/Oct/2023:06:08:18] ENGINE Bus STARTED 2023-10-09T06:08:19.516 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (2m) - 8m 13.8M - 0.23.0 ba2b418f427c 287b260dd8a4 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (87s) - 7m 40.6M - 8.3.5 dad864ee21e9 988e098b0939 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (6m) - 6m 14.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (6m) - 6m 13.1M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (35s) - 15m 67.7M - 18.2.0-598-gb4fa663f 3060897fdf97 6a90d5aaacfd 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (81s) - 17m 457M - 18.2.0-598-gb4fa663f 3060897fdf97 4731d4d85533 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (17m) - 17m 63.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 520280efd67b 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (15m) - 15m 60.5M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 920321cd9092 2023-10-09T06:08:19.517 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (15m) - 15m 58.8M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 7b9a79d51662 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (2m) - 8m 8951k - 1.3.1 1dbe0e931976 951da6650ff3 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (112s) - 8m 10.7M - 1.3.1 1dbe0e931976 4e060b6a5bef 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (14m) - 14m 44.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4276c724754b 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (13m) - 13m 45.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 67dcf3157f53 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (13m) - 13m 43.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 05b5c73536d5 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (12m) - 12m 42.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6d440ba42e37 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (11m) - 11m 41.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (11m) - 11m 43.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (10m) - 10m 44.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (9m) - 9m 47.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:08:19.518 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (41s) - 8m 42.8M - 2.33.4 514e6a882f6e 5a577340c8e2 2023-10-09T06:08:19.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[73481]: debug 2023-10-09T06:08:19.429+0000 7f8497396700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T06:08:19.636 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[72578]: level=info ts=2023-10-09T06:08:19.626Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2023-10-09T06:08:19.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.375720+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, since 63s), standbys: x 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.401556+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.402038+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.412162+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.412477+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.0106019s), standbys: x 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.416165+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.419301+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.426162+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.426880+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.427144+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:08:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.427363+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.428773+0000 mon.c (mon.2) 216 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.429178+0000 mon.c (mon.2) 217 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.429675+0000 mon.c (mon.2) 218 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.430160+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.430647+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.431074+0000 mon.c (mon.2) 221 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.431498+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.432040+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.432626+0000 mon.c (mon.2) 224 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:08:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.432972+0000 mon.c (mon.2) 225 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.433242+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.433666+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.434089+0000 mon.c (mon.2) 228 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.434464+0000 mon.c (mon.2) 229 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.435059+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.465223+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.502835+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.507380+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.543821+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.560133+0000 mon.c (mon.2) 234 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:08:19.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.560653+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:08:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.568407+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.590207+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: cluster 2023-10-09T06:08:18.597443+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.605634+0000 mon.c (mon.2) 235 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:08:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:19 smithi125 bash[33919]: audit 2023-10-09T06:08:18.606002+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:08:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.375720+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e32: y(active, since 63s), standbys: x 2023-10-09T06:08:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.401556+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-10-09T06:08:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.402038+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-10-09T06:08:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.412162+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.412477+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, starting, since 0.0106019s), standbys: x 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.416165+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.419301+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.426162+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.426880+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.427144+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.427363+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.428773+0000 mon.c (mon.2) 216 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi155.itxbmb"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.429178+0000 mon.c (mon.2) 217 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi125.yetyhy"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.429675+0000 mon.c (mon.2) 218 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:08:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.430160+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.430647+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.431074+0000 mon.c (mon.2) 221 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.431498+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.432040+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.432626+0000 mon.c (mon.2) 224 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.432972+0000 mon.c (mon.2) 225 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.433242+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:08:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.433666+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.434089+0000 mon.c (mon.2) 228 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.434464+0000 mon.c (mon.2) 229 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.435059+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.465223+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.502835+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.507380+0000 mon.c (mon.2) 232 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.543821+0000 mon.c (mon.2) 233 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.560133+0000 mon.c (mon.2) 234 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.560653+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:08:19.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.568407+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.590207+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: cluster 2023-10-09T06:08:18.597443+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e84 with expected crc 2023-10-09T06:08:19.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.605634+0000 mon.c (mon.2) 235 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:08:19.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:19 smithi155 bash[27817]: audit 2023-10-09T06:08:18.606002+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:08:20.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cephadm 2023-10-09T06:08:19.000889+0000 mgr.y (mgr.25723) 1 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Bus STARTING 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cephadm 2023-10-09T06:08:19.131789+0000 mgr.y (mgr.25723) 2 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cephadm 2023-10-09T06:08:19.233938+0000 mgr.y (mgr.25723) 3 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cephadm 2023-10-09T06:08:19.234253+0000 mgr.y (mgr.25723) 4 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Bus STARTED 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cluster 2023-10-09T06:08:19.423569+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e34: y(active, since 1.02168s), standbys: x 2023-10-09T06:08:20.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: audit 2023-10-09T06:08:19.425418+0000 mgr.y (mgr.25723) 5 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:08:20.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[33919]: cluster 2023-10-09T06:08:19.433391+0000 mgr.y (mgr.25723) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cephadm 2023-10-09T06:08:19.000889+0000 mgr.y (mgr.25723) 1 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Bus STARTING 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cephadm 2023-10-09T06:08:19.131789+0000 mgr.y (mgr.25723) 2 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cephadm 2023-10-09T06:08:19.233938+0000 mgr.y (mgr.25723) 3 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cephadm 2023-10-09T06:08:19.234253+0000 mgr.y (mgr.25723) 4 : cephadm [INF] [09/Oct/2023:06:08:19] ENGINE Bus STARTED 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cluster 2023-10-09T06:08:19.423569+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e34: y(active, since 1.02168s), standbys: x 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: audit 2023-10-09T06:08:19.425418+0000 mgr.y (mgr.25723) 5 : audit [DBG] from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:08:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:20 smithi155 bash[27817]: cluster 2023-10-09T06:08:19.433391+0000 mgr.y (mgr.25723) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:21.219 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:08:20 smithi125 bash[72578]: level=info ts=2023-10-09T06:08:20.937Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=6 2023-10-09T06:08:21.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:08:21] "GET /metrics HTTP/1.1" 200 31238 "" "Prometheus/2.33.4" 2023-10-09T06:08:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:21 smithi125 bash[33919]: cluster 2023-10-09T06:08:20.431926+0000 mgr.y (mgr.25723) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:21.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:21 smithi125 bash[33919]: cluster 2023-10-09T06:08:21.425923+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e35: y(active, since 3s), standbys: x 2023-10-09T06:08:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:21 smithi155 bash[27817]: cluster 2023-10-09T06:08:20.431926+0000 mgr.y (mgr.25723) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:21 smithi155 bash[27817]: cluster 2023-10-09T06:08:21.425923+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e35: y(active, since 3s), standbys: x 2023-10-09T06:08:23.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:23 smithi125 bash[33919]: cluster 2023-10-09T06:08:22.432450+0000 mgr.y (mgr.25723) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:23 smithi155 bash[27817]: cluster 2023-10-09T06:08:22.432450+0000 mgr.y (mgr.25723) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:24.166 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:08:25.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: audit 2023-10-09T06:08:24.155305+0000 mgr.y (mgr.25723) 9 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: cephadm 2023-10-09T06:08:24.156040+0000 mgr.y (mgr.25723) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: audit 2023-10-09T06:08:24.161358+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: cluster 2023-10-09T06:08:24.432967+0000 mgr.y (mgr.25723) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: audit 2023-10-09T06:08:25.047489+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:25.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:25 smithi125 bash[33919]: audit 2023-10-09T06:08:25.057351+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: audit 2023-10-09T06:08:24.155305+0000 mgr.y (mgr.25723) 9 : audit [DBG] from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: cephadm 2023-10-09T06:08:24.156040+0000 mgr.y (mgr.25723) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: audit 2023-10-09T06:08:24.161358+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: cluster 2023-10-09T06:08:24.432967+0000 mgr.y (mgr.25723) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: audit 2023-10-09T06:08:25.047489+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:25 smithi155 bash[27817]: audit 2023-10-09T06:08:25.057351+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:25.203701+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:25.212204+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:25.861735+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:25.872181+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.466 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:26.013755+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.466 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:26 smithi125 bash[33919]: audit 2023-10-09T06:08:26.023682+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:25.203701+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:25.212204+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:25.861735+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:25.872181+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:26.013755+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:26 smithi155 bash[27817]: audit 2023-10-09T06:08:26.023682+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:27.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:27 smithi125 bash[33919]: cluster 2023-10-09T06:08:26.433830+0000 mgr.y (mgr.25723) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:27 smithi155 bash[27817]: cluster 2023-10-09T06:08:26.433830+0000 mgr.y (mgr.25723) 12 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:29.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:08:28 smithi155 bash[54956]: ts=2023-10-09T06:08:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:08:29.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:29 smithi125 bash[33919]: cluster 2023-10-09T06:08:28.434619+0000 mgr.y (mgr.25723) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:29 smithi155 bash[27817]: cluster 2023-10-09T06:08:28.434619+0000 mgr.y (mgr.25723) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:31.496 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:08:31] "GET /metrics HTTP/1.1" 200 31238 "" "Prometheus/2.33.4" 2023-10-09T06:08:31.497 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:31 smithi125 bash[33919]: cluster 2023-10-09T06:08:30.435727+0000 mgr.y (mgr.25723) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:31 smithi155 bash[27817]: cluster 2023-10-09T06:08:30.435727+0000 mgr.y (mgr.25723) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:33.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:33 smithi125 bash[33919]: cluster 2023-10-09T06:08:32.436340+0000 mgr.y (mgr.25723) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:33.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:33 smithi155 bash[27817]: cluster 2023-10-09T06:08:32.436340+0000 mgr.y (mgr.25723) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:34 smithi125 bash[33919]: audit 2023-10-09T06:08:33.543764+0000 mon.c (mon.2) 236 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:34 smithi155 bash[27817]: audit 2023-10-09T06:08:33.543764+0000 mon.c (mon.2) 236 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:35 smithi125 bash[33919]: cluster 2023-10-09T06:08:34.437275+0000 mgr.y (mgr.25723) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:35 smithi125 bash[33919]: audit 2023-10-09T06:08:35.278608+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:35 smithi125 bash[33919]: audit 2023-10-09T06:08:35.287484+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:35.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:35 smithi125 bash[33919]: audit 2023-10-09T06:08:35.291739+0000 mon.c (mon.2) 237 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:35.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:35 smithi125 bash[33919]: audit 2023-10-09T06:08:35.293205+0000 mon.c (mon.2) 238 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:35 smithi155 bash[27817]: cluster 2023-10-09T06:08:34.437275+0000 mgr.y (mgr.25723) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:35 smithi155 bash[27817]: audit 2023-10-09T06:08:35.278608+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:35 smithi155 bash[27817]: audit 2023-10-09T06:08:35.287484+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:35 smithi155 bash[27817]: audit 2023-10-09T06:08:35.291739+0000 mon.c (mon.2) 237 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:35 smithi155 bash[27817]: audit 2023-10-09T06:08:35.293205+0000 mon.c (mon.2) 238 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:36.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.295064+0000 mgr.y (mgr.25723) 17 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.295458+0000 mgr.y (mgr.25723) 18 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.510173+0000 mgr.y (mgr.25723) 19 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.510573+0000 mgr.y (mgr.25723) 20 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.676738+0000 mgr.y (mgr.25723) 21 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.677092+0000 mgr.y (mgr.25723) 22 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.842079+0000 mgr.y (mgr.25723) 23 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: cephadm 2023-10-09T06:08:35.892074+0000 mgr.y (mgr.25723) 24 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.015665+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.024863+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.066927+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.077421+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.084765+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.167138+0000 mon.c (mon.2) 239 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.169988+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.171154+0000 mon.c (mon.2) 241 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:36 smithi125 bash[33919]: audit 2023-10-09T06:08:36.179771+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.295064+0000 mgr.y (mgr.25723) 17 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:08:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.295458+0000 mgr.y (mgr.25723) 18 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:08:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.510173+0000 mgr.y (mgr.25723) 19 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:08:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.510573+0000 mgr.y (mgr.25723) 20 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:08:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.676738+0000 mgr.y (mgr.25723) 21 : cephadm [INF] Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.677092+0000 mgr.y (mgr.25723) 22 : cephadm [INF] Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.842079+0000 mgr.y (mgr.25723) 23 : cephadm [INF] Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: cephadm 2023-10-09T06:08:35.892074+0000 mgr.y (mgr.25723) 24 : cephadm [INF] Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.015665+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.024863+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.066927+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.077421+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.084765+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.167138+0000 mon.c (mon.2) 239 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.169988+0000 mon.c (mon.2) 240 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.171154+0000 mon.c (mon.2) 241 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:36.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:36 smithi155 bash[27817]: audit 2023-10-09T06:08:36.179771+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:08:37.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:37 smithi125 bash[33919]: cluster 2023-10-09T06:08:36.437850+0000 mgr.y (mgr.25723) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:37 smithi155 bash[27817]: cluster 2023-10-09T06:08:36.437850+0000 mgr.y (mgr.25723) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:39.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:08:38 smithi155 bash[54956]: ts=2023-10-09T06:08:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:08:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:39 smithi125 bash[33919]: cluster 2023-10-09T06:08:38.438487+0000 mgr.y (mgr.25723) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:39 smithi155 bash[27817]: cluster 2023-10-09T06:08:38.438487+0000 mgr.y (mgr.25723) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:41.499 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:08:41] "GET /metrics HTTP/1.1" 200 33924 "" "Prometheus/2.33.4" 2023-10-09T06:08:41.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:41 smithi125 bash[33919]: cluster 2023-10-09T06:08:40.439572+0000 mgr.y (mgr.25723) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:41.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:41 smithi155 bash[27817]: cluster 2023-10-09T06:08:40.439572+0000 mgr.y (mgr.25723) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:43.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:43 smithi125 bash[33919]: cluster 2023-10-09T06:08:42.440206+0000 mgr.y (mgr.25723) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:43 smithi155 bash[27817]: cluster 2023-10-09T06:08:42.440206+0000 mgr.y (mgr.25723) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:45 smithi125 bash[33919]: cluster 2023-10-09T06:08:44.441248+0000 mgr.y (mgr.25723) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:45 smithi155 bash[27817]: cluster 2023-10-09T06:08:44.441248+0000 mgr.y (mgr.25723) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:47.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:47 smithi125 bash[33919]: cluster 2023-10-09T06:08:46.442018+0000 mgr.y (mgr.25723) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:47 smithi155 bash[27817]: cluster 2023-10-09T06:08:46.442018+0000 mgr.y (mgr.25723) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:49.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:08:48 smithi155 bash[54956]: ts=2023-10-09T06:08:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:08:49.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:49 smithi125 bash[33919]: cluster 2023-10-09T06:08:48.442748+0000 mgr.y (mgr.25723) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:49.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:49 smithi125 bash[33919]: audit 2023-10-09T06:08:48.544039+0000 mon.c (mon.2) 242 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:49 smithi155 bash[27817]: cluster 2023-10-09T06:08:48.442748+0000 mgr.y (mgr.25723) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:49 smithi155 bash[27817]: audit 2023-10-09T06:08:48.544039+0000 mon.c (mon.2) 242 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:51.504 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:08:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:08:51] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.33.4" 2023-10-09T06:08:51.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:51 smithi125 bash[33919]: cluster 2023-10-09T06:08:50.443753+0000 mgr.y (mgr.25723) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:51.912 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:51 smithi155 bash[27817]: cluster 2023-10-09T06:08:50.443753+0000 mgr.y (mgr.25723) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:53.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:53 smithi125 bash[33919]: cluster 2023-10-09T06:08:52.444391+0000 mgr.y (mgr.25723) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:53 smithi155 bash[27817]: cluster 2023-10-09T06:08:52.444391+0000 mgr.y (mgr.25723) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:55.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:55 smithi125 bash[33919]: cluster 2023-10-09T06:08:54.445314+0000 mgr.y (mgr.25723) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:55 smithi155 bash[27817]: cluster 2023-10-09T06:08:54.445314+0000 mgr.y (mgr.25723) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:57.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:57 smithi125 bash[33919]: cluster 2023-10-09T06:08:56.445953+0000 mgr.y (mgr.25723) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:57 smithi155 bash[27817]: cluster 2023-10-09T06:08:56.445953+0000 mgr.y (mgr.25723) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:59.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:08:58 smithi155 bash[54956]: ts=2023-10-09T06:08:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:08:59.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:08:59 smithi125 bash[33919]: cluster 2023-10-09T06:08:58.446622+0000 mgr.y (mgr.25723) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:08:59 smithi155 bash[27817]: cluster 2023-10-09T06:08:58.446622+0000 mgr.y (mgr.25723) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:01.508 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:01] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.33.4" 2023-10-09T06:09:01.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:01 smithi125 bash[33919]: cluster 2023-10-09T06:09:00.447691+0000 mgr.y (mgr.25723) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:01 smithi155 bash[27817]: cluster 2023-10-09T06:09:00.447691+0000 mgr.y (mgr.25723) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:03.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:03 smithi125 bash[33919]: cluster 2023-10-09T06:09:02.448418+0000 mgr.y (mgr.25723) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:03 smithi155 bash[27817]: cluster 2023-10-09T06:09:02.448418+0000 mgr.y (mgr.25723) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:04 smithi125 bash[33919]: audit 2023-10-09T06:09:03.543628+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:04 smithi155 bash[27817]: audit 2023-10-09T06:09:03.543628+0000 mon.c (mon.2) 243 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:05 smithi125 bash[33919]: cluster 2023-10-09T06:09:04.449416+0000 mgr.y (mgr.25723) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:05 smithi155 bash[27817]: cluster 2023-10-09T06:09:04.449416+0000 mgr.y (mgr.25723) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:07.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:07 smithi125 bash[33919]: cluster 2023-10-09T06:09:06.450031+0000 mgr.y (mgr.25723) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:07 smithi155 bash[27817]: cluster 2023-10-09T06:09:06.450031+0000 mgr.y (mgr.25723) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:09.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:08 smithi155 bash[54956]: ts=2023-10-09T06:09:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:09.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:09 smithi125 bash[33919]: cluster 2023-10-09T06:09:08.450638+0000 mgr.y (mgr.25723) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:09 smithi155 bash[27817]: cluster 2023-10-09T06:09:08.450638+0000 mgr.y (mgr.25723) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:11.512 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:11] "GET /metrics HTTP/1.1" 200 33923 "" "Prometheus/2.33.4" 2023-10-09T06:09:11.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:11 smithi125 bash[33919]: cluster 2023-10-09T06:09:10.451693+0000 mgr.y (mgr.25723) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:11.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:11 smithi155 bash[27817]: cluster 2023-10-09T06:09:10.451693+0000 mgr.y (mgr.25723) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:13 smithi125 bash[33919]: cluster 2023-10-09T06:09:12.452464+0000 mgr.y (mgr.25723) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:13 smithi155 bash[27817]: cluster 2023-10-09T06:09:12.452464+0000 mgr.y (mgr.25723) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:15 smithi125 bash[33919]: cluster 2023-10-09T06:09:14.453600+0000 mgr.y (mgr.25723) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:15 smithi155 bash[27817]: cluster 2023-10-09T06:09:14.453600+0000 mgr.y (mgr.25723) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:17.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:17 smithi125 bash[33919]: cluster 2023-10-09T06:09:16.454321+0000 mgr.y (mgr.25723) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:17.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:17 smithi155 bash[27817]: cluster 2023-10-09T06:09:16.454321+0000 mgr.y (mgr.25723) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:19.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:18 smithi155 bash[54956]: ts=2023-10-09T06:09:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:19 smithi125 bash[33919]: cluster 2023-10-09T06:09:18.455095+0000 mgr.y (mgr.25723) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:19.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:19 smithi125 bash[33919]: audit 2023-10-09T06:09:18.547744+0000 mon.c (mon.2) 244 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:19 smithi155 bash[27817]: cluster 2023-10-09T06:09:18.455095+0000 mgr.y (mgr.25723) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:19 smithi155 bash[27817]: audit 2023-10-09T06:09:18.547744+0000 mon.c (mon.2) 244 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:21.516 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:21] "GET /metrics HTTP/1.1" 200 33923 "" "Prometheus/2.33.4" 2023-10-09T06:09:21.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:21 smithi125 bash[33919]: cluster 2023-10-09T06:09:20.456481+0000 mgr.y (mgr.25723) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:21 smithi155 bash[27817]: cluster 2023-10-09T06:09:20.456481+0000 mgr.y (mgr.25723) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:23 smithi125 bash[33919]: cluster 2023-10-09T06:09:22.457144+0000 mgr.y (mgr.25723) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:23 smithi155 bash[27817]: cluster 2023-10-09T06:09:22.457144+0000 mgr.y (mgr.25723) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:25.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:25 smithi125 bash[33919]: cluster 2023-10-09T06:09:24.458147+0000 mgr.y (mgr.25723) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:25.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:25 smithi155 bash[27817]: cluster 2023-10-09T06:09:24.458147+0000 mgr.y (mgr.25723) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:27 smithi125 bash[33919]: cluster 2023-10-09T06:09:26.458816+0000 mgr.y (mgr.25723) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:27 smithi155 bash[27817]: cluster 2023-10-09T06:09:26.458816+0000 mgr.y (mgr.25723) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:29.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:09:29.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:28 smithi155 bash[54956]: ts=2023-10-09T06:09:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:29.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:29 smithi125 bash[33919]: cluster 2023-10-09T06:09:28.459382+0000 mgr.y (mgr.25723) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:29 smithi155 bash[27817]: cluster 2023-10-09T06:09:28.459382+0000 mgr.y (mgr.25723) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:31.521 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:31] "GET /metrics HTTP/1.1" 200 33923 "" "Prometheus/2.33.4" 2023-10-09T06:09:31.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:31 smithi125 bash[33919]: cluster 2023-10-09T06:09:30.460425+0000 mgr.y (mgr.25723) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:31 smithi155 bash[27817]: cluster 2023-10-09T06:09:30.460425+0000 mgr.y (mgr.25723) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:32.785 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:09:33.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T06:09:34.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: cluster 2023-10-09T06:09:32.461169+0000 mgr.y (mgr.25723) 53 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:34.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: audit 2023-10-09T06:09:32.773027+0000 mgr.y (mgr.25723) 54 : audit [DBG] from='client.26123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:09:34.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: cephadm 2023-10-09T06:09:32.773737+0000 mgr.y (mgr.25723) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:09:34.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: audit 2023-10-09T06:09:32.780483+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:34.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: audit 2023-10-09T06:09:32.823297+0000 mon.c (mon.2) 245 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:34.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:33 smithi125 bash[33919]: audit 2023-10-09T06:09:33.548346+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: cluster 2023-10-09T06:09:32.461169+0000 mgr.y (mgr.25723) 53 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: audit 2023-10-09T06:09:32.773027+0000 mgr.y (mgr.25723) 54 : audit [DBG] from='client.26123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:09:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: cephadm 2023-10-09T06:09:32.773737+0000 mgr.y (mgr.25723) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:09:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: audit 2023-10-09T06:09:32.780483+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: audit 2023-10-09T06:09:32.823297+0000 mon.c (mon.2) 245 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:33 smithi155 bash[27817]: audit 2023-10-09T06:09:33.548346+0000 mon.c (mon.2) 246 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:36.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:35 smithi125 bash[33919]: cluster 2023-10-09T06:09:34.462238+0000 mgr.y (mgr.25723) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:35 smithi155 bash[27817]: cluster 2023-10-09T06:09:34.462238+0000 mgr.y (mgr.25723) 56 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:36.804 INFO:teuthology.orchestra.run.smithi125.stdout:wait for majority of mons to upgrade 2023-10-09T06:09:37.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T06:09:38.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:37 smithi125 bash[33919]: cluster 2023-10-09T06:09:36.462900+0000 mgr.y (mgr.25723) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:38.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:37 smithi155 bash[27817]: cluster 2023-10-09T06:09:36.462900+0000 mgr.y (mgr.25723) 57 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:39.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:38 smithi155 bash[54956]: ts=2023-10-09T06:09:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:39 smithi125 bash[33919]: cluster 2023-10-09T06:09:38.463460+0000 mgr.y (mgr.25723) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:39 smithi125 bash[33919]: audit 2023-10-09T06:09:39.014265+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:39.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:39 smithi125 bash[33919]: audit 2023-10-09T06:09:39.025600+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:39 smithi155 bash[27817]: cluster 2023-10-09T06:09:38.463460+0000 mgr.y (mgr.25723) 58 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:39 smithi155 bash[27817]: audit 2023-10-09T06:09:39.014265+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:39 smithi155 bash[27817]: audit 2023-10-09T06:09:39.025600+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:39.773370+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:39.783448+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.253574+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.255397+0000 mon.c (mon.2) 248 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.266487+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.350138+0000 mon.c (mon.2) 249 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.354747+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.356509+0000 mon.c (mon.2) 251 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:09:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:40 smithi125 bash[33919]: audit 2023-10-09T06:09:40.357538+0000 mon.c (mon.2) 252 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:39.773370+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:39.783448+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.253574+0000 mon.c (mon.2) 247 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.255397+0000 mon.c (mon.2) 248 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.266487+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.350138+0000 mon.c (mon.2) 249 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.354747+0000 mon.c (mon.2) 250 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.356509+0000 mon.c (mon.2) 251 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:09:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:40 smithi155 bash[27817]: audit 2023-10-09T06:09:40.357538+0000 mon.c (mon.2) 252 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T06:09:41.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:41] "GET /metrics HTTP/1.1" 200 33922 "" "Prometheus/2.33.4" 2023-10-09T06:09:41.946 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:41 smithi125 bash[33919]: cephadm 2023-10-09T06:09:40.358315+0000 mgr.y (mgr.25723) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T06:09:41.947 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:41 smithi125 bash[33919]: cluster 2023-10-09T06:09:40.464312+0000 mgr.y (mgr.25723) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:41.947 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:41 smithi125 bash[33919]: audit 2023-10-09T06:09:41.623093+0000 mon.a (mon.0) 46 : audit [DBG] from='client.? 172.21.15.125:0/3116612886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:41 smithi155 bash[27817]: cephadm 2023-10-09T06:09:40.358315+0000 mgr.y (mgr.25723) 59 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-09T06:09:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:41 smithi155 bash[27817]: cluster 2023-10-09T06:09:40.464312+0000 mgr.y (mgr.25723) 60 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:41 smithi155 bash[27817]: audit 2023-10-09T06:09:41.623093+0000 mon.a (mon.0) 46 : audit [DBG] from='client.? 172.21.15.125:0/3116612886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:44.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: cluster 2023-10-09T06:09:42.464912+0000 mgr.y (mgr.25723) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: cephadm 2023-10-09T06:09:42.777157+0000 mgr.y (mgr.25723) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: audit 2023-10-09T06:09:42.788149+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: audit 2023-10-09T06:09:42.789733+0000 mon.c (mon.2) 253 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: audit 2023-10-09T06:09:42.791337+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: audit 2023-10-09T06:09:42.792514+0000 mon.c (mon.2) 254 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:44.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:43 smithi125 bash[33919]: cephadm 2023-10-09T06:09:42.793877+0000 mgr.y (mgr.25723) 63 : cephadm [INF] Deploying daemon mon.c on smithi125 2023-10-09T06:09:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: cluster 2023-10-09T06:09:42.464912+0000 mgr.y (mgr.25723) 61 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: cephadm 2023-10-09T06:09:42.777157+0000 mgr.y (mgr.25723) 62 : cephadm [INF] Upgrade: Updating mon.c 2023-10-09T06:09:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: audit 2023-10-09T06:09:42.788149+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25723 ' entity='mgr.y' 2023-10-09T06:09:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: audit 2023-10-09T06:09:42.789733+0000 mon.c (mon.2) 253 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:09:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: audit 2023-10-09T06:09:42.791337+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.25723 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:09:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: audit 2023-10-09T06:09:42.792514+0000 mon.c (mon.2) 254 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:43 smithi155 bash[27817]: cephadm 2023-10-09T06:09:42.793877+0000 mgr.y (mgr.25723) 63 : cephadm [INF] Deploying daemon mon.c on smithi125 2023-10-09T06:09:45.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:44 smithi125 bash[33919]: audit 2023-10-09T06:09:44.077893+0000 mon.c (mon.2) 255 : audit [DBG] from='client.? 172.21.15.125:0/3061807302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:45.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:44 smithi155 bash[27817]: audit 2023-10-09T06:09:44.077893+0000 mon.c (mon.2) 255 : audit [DBG] from='client.? 172.21.15.125:0/3061807302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE Bus STOPPING 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE Bus STOPPED 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE Bus STARTING 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:45 smithi125 systemd[1]: Stopping Ceph mon.c for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[33919]: debug 2023-10-09T06:09:45.288+0000 7f97d6a19700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:09:45.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[33919]: debug 2023-10-09T06:09:45.288+0000 7f97d6a19700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T06:09:45.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE Serving on http://:::9283 2023-10-09T06:09:45.886 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:45 smithi125 bash[73481]: [09/Oct/2023:06:09:45] ENGINE Bus STARTED 2023-10-09T06:09:46.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:46 smithi125 bash[79847]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T06:09:46.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:46 smithi125 bash[79945]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T06:09:46.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:46 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T06:09:46.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:46 smithi125 systemd[1]: Stopped Ceph mon.c for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:09:46.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:46 smithi125 systemd[1]: Started Ceph mon.c for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.743+0000 7fb872aa1c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.743+0000 7fb872aa1c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.743+0000 7fb872aa1c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.751+0000 7fb872aa1c80 0 load: jerasure load: lrc 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T06:09:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Git sha 0 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: DB SUMMARY 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: DB Session ID: 016D3EK1FYLS6WIRQJFV 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1276 Bytes 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 970447 ; 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T06:09:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.env: 0x558b6c8be720 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.info_log: 0x558b6ef89480 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T06:09:47.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T06:09:47.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.db_log_dir: 2023-10-09T06:09:47.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.wal_dir: 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.write_buffer_manager: 0x558b6e216c80 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:09:47.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:09:47.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.row_cache: None 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.wal_filter: None 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:09:47.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:09:47.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:09:47.896 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Compression algorithms supported: 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kZSTD supported: 0 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-09T06:09:47.897 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.merge_operator: 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558b6ef89580) 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: cache_index_and_filter_blocks: 1 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:09:47.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: pin_top_level_index_and_filter: 1 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: index_type: 0 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: data_block_index_type: 0 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: index_shortening: 1 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: checksum: 4 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: no_block_cache: 0 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_cache: 0x558b6e2991f0 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_cache_name: BinnedLRUCache 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_cache_options: 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: capacity : 536870912 2023-10-09T06:09:47.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: num_shard_bits : 4 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: strict_capacity_limit : 0 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: high_pri_pool_ratio: 0.000 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_cache_compressed: (nil) 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: persistent_cache: (nil) 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_size: 4096 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_size_deviation: 10 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_restart_interval: 16 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: index_block_restart_interval: 1 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: metadata_block_size: 4096 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: partition_filters: 0 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: use_delta_encoding: 1 2023-10-09T06:09:47.900 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: filter_policy: bloomfilter 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: whole_key_filtering: 1 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: verify_compression: 0 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: read_amp_bytes_per_bit: 0 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: format_version: 5 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: enable_index_compression: 1 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: block_align: 0 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: max_auto_readahead_size: 262144 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: prepopulate_block_cache: 0 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: initial_auto_readahead_size: 8192 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: num_file_reads_for_auto_readahead: 2 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:09:47.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.num_levels: 7 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:09:47.902 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:09:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:09:47.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:09:47.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:09:47.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:09:47.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:09:47.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb8682bf700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 17874, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:09:47.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5b9e362d-5412-4253-af70-d264a82a1097 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831787761593, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.755+0000 7fb872aa1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831787767594, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 645689, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17869, "largest_seqno": 18068, "table_properties": {"data_size": 643587, "index_size": 898, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 197, "raw_key_size": 1709, "raw_average_key_size": 23, "raw_value_size": 641397, "raw_average_value_size": 8667, "num_data_blocks": 38, "num_entries": 74, "num_filter_entries": 74, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831787, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5b9e362d-5412-4253-af70-d264a82a1097", "db_session_id": "016D3EK1FYLS6WIRQJFV", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831787767708, "job": 1, "event": "recovery_finished"} 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558b6e33a000 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb872aa1c80 4 rocksdb: DB pointer 0x558b6e328000 2023-10-09T06:09:47.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb8612b1700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.763+0000 7fb8612b1700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: ** DB Stats ** 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: ** Compaction Stats [default] ** 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:09:47.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: L0 1/0 630.56 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 193.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: L6 1/0 32.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Sum 2/0 32.86 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 193.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 193.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: ** Compaction Stats [default] ** 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 193.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T06:09:47.912 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Cumulative compaction: 0.00 GB write, 58.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Interval compaction: 0.00 GB write, 58.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Block cache BinnedLRUCache@0x558b6e2991f0#7 capacity: 512.00 MB usage: 1.31 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Block cache entry stats(count,size,portion): FilterBlock(1,0.23 KB,4.47035e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] at bind addrs [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 1 mon.c@-1(???) e3 preinit fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:09:47.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).mds e7 new map 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).mds e7 print_map 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: e7 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: legacy client fscid: 1 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Filesystem 'foo' (1) 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: fs_name foo 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: epoch 5 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T06:09:47.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: created 2023-10-09T06:01:50.652169+0000 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: modified 2023-10-09T06:01:58.450626+0000 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: tableserver 0 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: root 0 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: session_timeout 60 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: session_autoclose 300 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: max_file_size 1099511627776 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: required_client_features {} 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: last_failure 0 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: last_failure_osd_epoch 0 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: max_mds 1 2023-10-09T06:09:47.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: in 0 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: up {0=24542} 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: failed 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: damaged 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: stopped 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: data_pools [4] 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: metadata_pool 3 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: inline_data disabled 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: balancer 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: bal_rank_mask -1 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: standby_count_wanted 1 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: [mds.foo.smithi125.yetyhy{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:09:47.916 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: Standby daemons: 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: [mds.foo.smithi155.itxbmb{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 0 mon.c@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.767+0000 7fb872aa1c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T06:09:47.917 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T06:09:47.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:09:47 smithi125 bash[80002]: debug 2023-10-09T06:09:47.771+0000 7fb872aa1c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T06:09:49.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:48 smithi155 bash[54956]: ts=2023-10-09T06:09:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:51.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:09:51] "GET /metrics HTTP/1.1" 200 33922 "" "Prometheus/2.33.4" 2023-10-09T06:09:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:52 smithi155 bash[27817]: cluster 2023-10-09T06:09:44.465795+0000 mgr.y (mgr.25723) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:52 smithi155 bash[27817]: audit 2023-10-09T06:09:45.304359+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:09:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:52 smithi155 bash[27817]: audit 2023-10-09T06:09:45.304586+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:09:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:53 smithi155 bash[27817]: audit 2023-10-09T06:09:45.304778+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:09:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:46.466346+0000 mgr.y (mgr.25723) 65 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:46.546664+0000 mon.a (mon.0) 52 : audit [DBG] from='client.? 172.21.15.125:0/137748366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:47.987268+0000 mon.b (mon.1) 101 : cluster [INF] mon.b calling monitor election 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:47.988396+0000 mon.a (mon.0) 53 : cluster [INF] mon.a calling monitor election 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:48.466923+0000 mgr.y (mgr.25723) 66 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:48.548632+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:50.467964+0000 mgr.y (mgr.25723) 67 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:52.468692+0000 mgr.y (mgr.25723) 68 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:52.991171+0000 mon.a (mon.0) 55 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.000853+0000 mon.a (mon.0) 56 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.001991+0000 mon.a (mon.0) 57 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.002048+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.002548+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e35: y(active, since 94s), standbys: x 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.002795+0000 mon.a (mon.0) 60 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-09T06:09:54.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.028688+0000 mon.a (mon.0) 61 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.028722+0000 mon.a (mon.0) 62 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: cluster 2023-10-09T06:09:53.028740+0000 mon.a (mon.0) 63 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0] is down (out of quorum) 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:53.031672+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:53.038812+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:53.084121+0000 mon.a (mon.0) 66 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:54.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:54 smithi155 bash[27817]: audit 2023-10-09T06:09:53.415664+0000 mon.a (mon.0) 67 : audit [DBG] from='client.? 172.21.15.125:0/3443539135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:47.984500+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-09T06:09:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.006273+0000 mon.a (mon.0) 68 : cluster [INF] mon.a calling monitor election 2023-10-09T06:09:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.007483+0000 mon.b (mon.1) 102 : cluster [INF] mon.b calling monitor election 2023-10-09T06:09:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.045139+0000 mon.a (mon.0) 69 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.068811+0000 mon.a (mon.0) 70 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.087458+0000 mon.a (mon.0) 71 : cluster [DBG] fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.087548+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.088249+0000 mon.a (mon.0) 73 : cluster [DBG] mgrmap e36: y(active, since 95s), standbys: x 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.088821+0000 mon.a (mon.0) 74 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.088883+0000 mon.a (mon.0) 75 : cluster [INF] Cluster is now healthy 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.096032+0000 mon.a (mon.0) 76 : cluster [INF] overall HEALTH_OK 2023-10-09T06:09:55.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:55 smithi155 bash[27817]: cluster 2023-10-09T06:09:54.469847+0000 mgr.y (mgr.25723) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:09:55.887 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T06:09:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:56 smithi155 bash[27817]: audit 2023-10-09T06:09:55.866200+0000 mon.a (mon.0) 77 : audit [DBG] from='client.? 172.21.15.125:0/2363395131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:09:56.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:09:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:57 smithi155 bash[27817]: cluster 2023-10-09T06:09:56.470653+0000 mgr.y (mgr.25723) 70 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:09:58.076 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:09:57 smithi125 bash[73481]: debug 2023-10-09T06:09:57.775+0000 7f8497396700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T06:09:59.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:09:58 smithi155 bash[54956]: ts=2023-10-09T06:09:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:09:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:59 smithi155 bash[27817]: cluster 2023-10-09T06:09:58.471021+0000 mgr.y (mgr.25723) 71 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:09:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:59 smithi155 bash[27817]: audit 2023-10-09T06:09:59.026393+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:09:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:09:59 smithi155 bash[27817]: audit 2023-10-09T06:09:59.036703+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:00.318 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (3m) - 9m 13.7M - 0.23.0 ba2b418f427c 287b260dd8a4 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (3m) - 9m 41.0M - 8.3.5 dad864ee21e9 988e098b0939 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (8m) - 8m 15.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (8m) - 8m 13.5M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (2m) - 16m 426M - 18.2.0-598-gb4fa663f 3060897fdf97 6a90d5aaacfd 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (3m) - 18m 474M - 18.2.0-598-gb4fa663f 3060897fdf97 4731d4d85533 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (113s) - 19m 49.6M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 66d70cab6c5d 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (17m) - 17m 57.2M 2048M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 920321cd9092 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (13s) - 17m 19.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 bedca0a9cc9d 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (3m) - 10m 8947k - 1.3.1 1dbe0e931976 951da6650ff3 2023-10-09T06:10:00.319 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (3m) - 10m 9192k - 1.3.1 1dbe0e931976 4e060b6a5bef 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (16m) - 16m 45.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4276c724754b 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (15m) - 15m 46.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 67dcf3157f53 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (14m) - 14m 43.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 05b5c73536d5 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (14m) - 14m 43.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6d440ba42e37 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (13m) - 13m 42.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (12m) - 12m 43.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (11m) - 12m 44.6M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (11m) - 11m 47.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:10:00.320 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (2m) - 10m 53.4M - 2.33.4 514e6a882f6e 5a577340c8e2 2023-10-09T06:10:01.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:10:01.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:00 smithi155 bash[27817]: audit 2023-10-09T06:09:59.887082+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:01.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:00 smithi155 bash[27817]: audit 2023-10-09T06:09:59.896749+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:01.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:00 smithi155 bash[27817]: cluster 2023-10-09T06:10:00.000164+0000 mon.a (mon.0) 82 : cluster [INF] overall HEALTH_OK 2023-10-09T06:10:01.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:01] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.33.4" 2023-10-09T06:10:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:01 smithi155 bash[27817]: audit 2023-10-09T06:10:00.297843+0000 mgr.y (mgr.25723) 72 : audit [DBG] from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:10:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:01 smithi155 bash[27817]: cluster 2023-10-09T06:10:00.472118+0000 mgr.y (mgr.25723) 73 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:10:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:03 smithi155 bash[27817]: cluster 2023-10-09T06:10:02.472762+0000 mgr.y (mgr.25723) 74 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:03 smithi155 bash[27817]: audit 2023-10-09T06:10:03.549109+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:04.690 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:10:05.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:10:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:05 smithi155 bash[27817]: cluster 2023-10-09T06:10:04.473860+0000 mgr.y (mgr.25723) 75 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:05 smithi155 bash[27817]: audit 2023-10-09T06:10:04.680643+0000 mgr.y (mgr.25723) 76 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:10:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:05 smithi155 bash[27817]: cephadm 2023-10-09T06:10:04.681237+0000 mgr.y (mgr.25723) 77 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:10:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:05 smithi155 bash[27817]: audit 2023-10-09T06:10:04.687417+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:08.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:07 smithi155 bash[27817]: cluster 2023-10-09T06:10:06.474546+0000 mgr.y (mgr.25723) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:09.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:08 smithi155 bash[54956]: ts=2023-10-09T06:10:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:09 smithi155 bash[27817]: cluster 2023-10-09T06:10:08.475164+0000 mgr.y (mgr.25723) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:10.393 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:10 smithi155 bash[54956]: ts=2023-10-09T06:10:10.088Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", public_addr=\"172.21.15.125\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.125\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-09T06:10:11.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:11] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-10-09T06:10:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:11 smithi155 bash[27817]: cluster 2023-10-09T06:10:10.476093+0000 mgr.y (mgr.25723) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.548108+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.556169+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.557954+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.559176+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.565280+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.648236+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.651526+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.652853+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:12 smithi155 bash[27817]: audit 2023-10-09T06:10:11.663305+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:10:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:13 smithi155 bash[27817]: cluster 2023-10-09T06:10:12.476689+0000 mgr.y (mgr.25723) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:15 smithi155 bash[27817]: cluster 2023-10-09T06:10:14.477690+0000 mgr.y (mgr.25723) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:17 smithi155 bash[27817]: cluster 2023-10-09T06:10:16.478368+0000 mgr.y (mgr.25723) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:19.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:18 smithi155 bash[54956]: ts=2023-10-09T06:10:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:19 smithi155 bash[27817]: cluster 2023-10-09T06:10:18.479096+0000 mgr.y (mgr.25723) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:19 smithi155 bash[27817]: audit 2023-10-09T06:10:18.549438+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:21.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:21] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-10-09T06:10:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:21 smithi155 bash[27817]: cluster 2023-10-09T06:10:20.480190+0000 mgr.y (mgr.25723) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:23 smithi155 bash[27817]: cluster 2023-10-09T06:10:22.480757+0000 mgr.y (mgr.25723) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:25 smithi155 bash[27817]: cluster 2023-10-09T06:10:24.481832+0000 mgr.y (mgr.25723) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:27 smithi155 bash[27817]: cluster 2023-10-09T06:10:26.482481+0000 mgr.y (mgr.25723) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:29.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:28 smithi155 bash[54956]: ts=2023-10-09T06:10:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:29 smithi155 bash[27817]: cluster 2023-10-09T06:10:28.483091+0000 mgr.y (mgr.25723) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:31.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:31] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-10-09T06:10:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:31 smithi155 bash[27817]: cluster 2023-10-09T06:10:30.484109+0000 mgr.y (mgr.25723) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:33 smithi155 bash[27817]: cluster 2023-10-09T06:10:32.484717+0000 mgr.y (mgr.25723) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:34 smithi155 bash[27817]: audit 2023-10-09T06:10:33.551982+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:35 smithi155 bash[27817]: cluster 2023-10-09T06:10:34.485685+0000 mgr.y (mgr.25723) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:37 smithi155 bash[27817]: cluster 2023-10-09T06:10:36.486345+0000 mgr.y (mgr.25723) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:39.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:38 smithi155 bash[54956]: ts=2023-10-09T06:10:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:39 smithi155 bash[27817]: cluster 2023-10-09T06:10:38.486980+0000 mgr.y (mgr.25723) 94 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:41.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:41] "GET /metrics HTTP/1.1" 200 33925 "" "Prometheus/2.33.4" 2023-10-09T06:10:41.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:41 smithi155 bash[27817]: cluster 2023-10-09T06:10:40.488111+0000 mgr.y (mgr.25723) 95 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:43 smithi155 bash[27817]: cluster 2023-10-09T06:10:42.488823+0000 mgr.y (mgr.25723) 96 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:45 smithi155 bash[27817]: cluster 2023-10-09T06:10:44.489924+0000 mgr.y (mgr.25723) 97 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:47 smithi155 bash[27817]: cluster 2023-10-09T06:10:46.490619+0000 mgr.y (mgr.25723) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:49.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:48 smithi155 bash[54956]: ts=2023-10-09T06:10:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:49 smithi155 bash[27817]: cluster 2023-10-09T06:10:48.491261+0000 mgr.y (mgr.25723) 99 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:49 smithi155 bash[27817]: audit 2023-10-09T06:10:48.550271+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:51.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:10:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:10:51] "GET /metrics HTTP/1.1" 200 33923 "" "Prometheus/2.33.4" 2023-10-09T06:10:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:51 smithi155 bash[27817]: cluster 2023-10-09T06:10:50.492405+0000 mgr.y (mgr.25723) 100 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:53 smithi155 bash[27817]: cluster 2023-10-09T06:10:52.493110+0000 mgr.y (mgr.25723) 101 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:55.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:55 smithi155 bash[27817]: cluster 2023-10-09T06:10:54.494198+0000 mgr.y (mgr.25723) 102 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:57 smithi155 bash[27817]: cluster 2023-10-09T06:10:56.494914+0000 mgr.y (mgr.25723) 103 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:10:59.143 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:10:58 smithi155 bash[54956]: ts=2023-10-09T06:10:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:10:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:10:59 smithi155 bash[27817]: cluster 2023-10-09T06:10:58.495690+0000 mgr.y (mgr.25723) 104 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:01.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:01] "GET /metrics HTTP/1.1" 200 33923 "" "Prometheus/2.33.4" 2023-10-09T06:11:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:01 smithi155 bash[27817]: cluster 2023-10-09T06:11:00.496663+0000 mgr.y (mgr.25723) 105 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:03 smithi155 bash[27817]: cluster 2023-10-09T06:11:02.497266+0000 mgr.y (mgr.25723) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:03 smithi155 bash[27817]: audit 2023-10-09T06:11:03.550630+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:05 smithi155 bash[27817]: cluster 2023-10-09T06:11:04.498469+0000 mgr.y (mgr.25723) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:07 smithi155 bash[27817]: cluster 2023-10-09T06:11:06.499114+0000 mgr.y (mgr.25723) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:09.144 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:11:08 smithi155 bash[54956]: ts=2023-10-09T06:11:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:11:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:09 smithi155 bash[27817]: cluster 2023-10-09T06:11:08.499785+0000 mgr.y (mgr.25723) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:11.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:11] "GET /metrics HTTP/1.1" 200 33854 "" "Prometheus/2.33.4" 2023-10-09T06:11:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:11 smithi155 bash[27817]: cluster 2023-10-09T06:11:10.500842+0000 mgr.y (mgr.25723) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:11.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:11:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:12 smithi155 bash[27817]: audit 2023-10-09T06:11:11.747260+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:13 smithi155 bash[27817]: cluster 2023-10-09T06:11:12.501490+0000 mgr.y (mgr.25723) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:15.187 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:15 smithi155 bash[27817]: cluster 2023-10-09T06:11:14.502626+0000 mgr.y (mgr.25723) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:15 smithi155 bash[27817]: audit 2023-10-09T06:11:15.183795+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:15.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T06:11:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:16 smithi155 bash[27817]: audit 2023-10-09T06:11:15.176977+0000 mgr.y (mgr.25723) 113 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:16 smithi155 bash[27817]: cephadm 2023-10-09T06:11:15.177588+0000 mgr.y (mgr.25723) 114 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:17 smithi155 bash[27817]: cluster 2023-10-09T06:11:16.503218+0000 mgr.y (mgr.25723) 115 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:18.955 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[54956]: ts=2023-10-09T06:11:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi155.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)\", cluster_addr=\"172.21.15.125\", device_class=\"ssd\", hostname=\"smithi125\", instance=\"smithi125:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.125\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:11:18.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:17.745083+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:18.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:17.754642+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:18.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.044758+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:18.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.050522+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:18.955 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.520925+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.522613+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.530032+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.550142+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.641619+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.645863+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.647911+0000 mon.a (mon.0) 110 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:11:18.956 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:18 smithi155 bash[27817]: audit 2023-10-09T06:11:18.649198+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:11:19.117 INFO:teuthology.orchestra.run.smithi125.stdout:wait for all mons to upgrade 2023-10-09T06:11:20.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T06:11:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:19 smithi155 bash[27817]: cluster 2023-10-09T06:11:18.503762+0000 mgr.y (mgr.25723) 116 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:19 smithi155 bash[27817]: cephadm 2023-10-09T06:11:18.650022+0000 mgr.y (mgr.25723) 117 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-09T06:11:21.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:21] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.33.4" 2023-10-09T06:11:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:21 smithi155 bash[27817]: cluster 2023-10-09T06:11:20.504758+0000 mgr.y (mgr.25723) 118 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:21.911 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:21 smithi155 bash[27817]: audit 2023-10-09T06:11:21.328636+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:21.912 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:21 smithi155 bash[27817]: audit 2023-10-09T06:11:21.329960+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:21.912 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:21 smithi155 bash[27817]: audit 2023-10-09T06:11:21.330972+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:21.912 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:21 smithi155 bash[27817]: audit 2023-10-09T06:11:21.331918+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:22 smithi155 bash[27817]: cephadm 2023-10-09T06:11:21.320724+0000 mgr.y (mgr.25723) 119 : cephadm [INF] Upgrade: Updating mon.b 2023-10-09T06:11:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:22 smithi155 bash[27817]: cephadm 2023-10-09T06:11:21.333142+0000 mgr.y (mgr.25723) 120 : cephadm [INF] Deploying daemon mon.b on smithi155 2023-10-09T06:11:23.765 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:23 smithi155 systemd[1]: Stopping Ceph mon.b for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:11:24.120 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:23 smithi155 bash[27817]: cluster 2023-10-09T06:11:22.505338+0000 mgr.y (mgr.25723) 121 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:24.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:23 smithi155 bash[27817]: audit 2023-10-09T06:11:23.497762+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.125:0/2832843321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:24.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:23 smithi155 bash[27817]: debug 2023-10-09T06:11:23.842+0000 7f389fe0b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:11:24.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:23 smithi155 bash[27817]: debug 2023-10-09T06:11:23.842+0000 7f389fe0b700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T06:11:24.891 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:24 smithi155 bash[59363]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T06:11:24.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:24 smithi155 bash[59425]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T06:11:24.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:24 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T06:11:24.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:24 smithi155 systemd[1]: Stopped Ceph mon.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:11:25.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:24 smithi155 systemd[1]: Started Ceph mon.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.006+0000 7fd7e92e3c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.006+0000 7fd7e92e3c80 0 ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.006+0000 7fd7e92e3c80 0 pidfile_write: ignore empty --pid-file 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 0 load: jerasure load: lrc 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Git sha 0 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: DB SUMMARY 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: DB Session ID: UAURTZT45LSP01TOX0UW 2023-10-09T06:11:26.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: CURRENT file: CURRENT 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1662 Bytes 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000032.sst 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000030.log size: 1694512 ; 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.error_if_exists: 0 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.create_if_missing: 0 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:11:26.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.env: 0x55be9f0f0720 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.info_log: 0x55bea0a91480 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.statistics: (nil) 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.use_fsync: 0 2023-10-09T06:11:26.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-09T06:11:26.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:11:26.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:11:26.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:11:26.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:11:26.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.db_log_dir: 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.wal_dir: 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:11:26.148 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.write_buffer_manager: 0x55be9fd1ec80 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:11:26.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.unordered_write: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.row_cache: None 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.wal_filter: None 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.two_write_queues: 0 2023-10-09T06:11:26.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.wal_compression: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.atomic_flush: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:11:26.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:11:26.153 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_open_files: -1 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-09T06:11:26.154 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Compression algorithms supported: 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kZSTD supported: 0 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kXpressCompression supported: 0 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kZlibCompression supported: 1 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-09T06:11:26.155 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.merge_operator: 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_filter: None 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bea0a91580) 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: cache_index_and_filter_blocks: 1 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:11:26.156 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: pin_top_level_index_and_filter: 1 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: index_type: 0 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: data_block_index_type: 0 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: index_shortening: 1 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: checksum: 4 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: no_block_cache: 0 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_cache: 0x55be9fda1350 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_cache_name: BinnedLRUCache 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_cache_options: 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: capacity : 536870912 2023-10-09T06:11:26.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: num_shard_bits : 4 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: strict_capacity_limit : 0 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: high_pri_pool_ratio: 0.000 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_cache_compressed: (nil) 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: persistent_cache: (nil) 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_size: 4096 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_size_deviation: 10 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_restart_interval: 16 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: index_block_restart_interval: 1 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: metadata_block_size: 4096 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: partition_filters: 0 2023-10-09T06:11:26.158 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: use_delta_encoding: 1 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: filter_policy: bloomfilter 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: whole_key_filtering: 1 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: verify_compression: 0 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: read_amp_bytes_per_bit: 0 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: format_version: 5 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: enable_index_compression: 1 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: block_align: 0 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: max_auto_readahead_size: 262144 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: prepopulate_block_cache: 0 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: initial_auto_readahead_size: 8192 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: num_file_reads_for_auto_readahead: 2 2023-10-09T06:11:26.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression: NoCompression 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.num_levels: 7 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:11:26.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:11:26.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:11:26.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:11:26.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:11:26.164 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.table_properties_collectors: 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-09T06:11:26.165 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.bloom_locality: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.ttl: 2592000 2023-10-09T06:11:26.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enable_blob_files: false 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.min_blob_size: 0 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:11:26.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.010+0000 7fd7e92e3c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7deb01700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 32.sst 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7e92e3c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 34, last_sequence is 19857, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7e92e3c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7e92e3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6ae5e021-d8ea-46ce-882f-a154c601a132 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7e92e3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831886018685, "job": 1, "event": "recovery_started", "wal_files": [30]} 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.014+0000 7fd7e92e3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.018+0000 7fd7e92e3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831886024023, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 1126679, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19848, "largest_seqno": 20175, "table_properties": {"data_size": 1123914, "index_size": 1433, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2446, "raw_average_key_size": 21, "raw_value_size": 1120624, "raw_average_value_size": 10005, "num_data_blocks": 62, "num_entries": 112, "num_filter_entries": 112, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831886, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6ae5e021-d8ea-46ce-882f-a154c601a132", "db_session_id": "UAURTZT45LSP01TOX0UW", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.018+0000 7fd7e92e3c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696831886024132, "job": 1, "event": "recovery_finished"} 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.018+0000 7fd7e92e3c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 37 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7e92e3c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:11:26.168 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7e92e3c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55be9fe42000 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7e92e3c80 4 rocksdb: DB pointer 0x55be9fe30000 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7d7af3700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7d7af3700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: ** DB Stats ** 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: ** Compaction Stats [default] ** 2023-10-09T06:11:26.169 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: L0 1/0 1.07 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: L6 1/0 29.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Sum 2/0 30.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 396.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: ** Compaction Stats [default] ** 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 396.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:11:26.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Cumulative compaction: 0.00 GB write, 103.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Interval compaction: 0.00 GB write, 103.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Block cache BinnedLRUCache@0x55be9fda1350#7 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2023-10-09T06:11:26.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7e92e3c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] at bind addrs [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.022+0000 7fd7e92e3c80 1 mon.b@-1(???) e3 preinit fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).mds e7 new map 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).mds e7 print_map 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: e7 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: legacy client fscid: 1 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Filesystem 'foo' (1) 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: fs_name foo 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: epoch 5 2023-10-09T06:11:26.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: created 2023-10-09T06:01:50.652169+0000 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: modified 2023-10-09T06:01:58.450626+0000 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: tableserver 0 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: root 0 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: session_timeout 60 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: session_autoclose 300 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: max_file_size 1099511627776 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: required_client_features {} 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: last_failure 0 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: last_failure_osd_epoch 0 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:11:26.173 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: max_mds 1 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: in 0 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: up {0=24542} 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: failed 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: damaged 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: stopped 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: data_pools [4] 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: metadata_pool 3 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: inline_data disabled 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: balancer 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: bal_rank_mask -1 2023-10-09T06:11:26.174 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: standby_count_wanted 1 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: [mds.foo.smithi125.yetyhy{0:24542} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.125:6834/3928432442,v1:172.21.15.125:6835/3928432442] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: Standby daemons: 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: [mds.foo.smithi155.itxbmb{-1:24502} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.155:6832/1288601725,v1:172.21.15.155:6833/1288601725] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 0 mon.b@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-10-09T06:11:26.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expand map: {default=false} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta from 'false' to 'false' 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expanded map: {default=false} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expand map: {default=info} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta from 'info' to 'info' 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expanded map: {default=info} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expand map: {default=daemon} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expanded map: {default=daemon} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expand map: {default=debug} 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta from 'debug' to 'debug' 2023-10-09T06:11:26.176 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:26 smithi155 bash[59495]: debug 2023-10-09T06:11:26.026+0000 7fd7e92e3c80 20 expand_channel_meta expanded map: {default=debug} 2023-10-09T06:11:27.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T05:50:59.273186+0000 mon.a (mon.0) 0 : 1 mkfs 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:11:27.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm 1 Deploying daemon mgr.y on smithi125 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:11:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T05:50:59.273186+0000 mon.a (mon.0) 0 : 1 mkfs 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:11:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm 1 Deploying daemon mgr.y on smithi125 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm 1 Deploying daemon mgr.x on smithi155 2023-10-09T06:11:27.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm 1 Deploying daemon mon.a on smithi125 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:19.000889+0000 mgr.y (mgr.25723) 1 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTING 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:19.131789+0000 mgr.y (mgr.25723) 2 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:19.233938+0000 mgr.y (mgr.25723) 3 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:19.234253+0000 mgr.y (mgr.25723) 4 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTED 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:24.156040+0000 mgr.y (mgr.25723) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.295064+0000 mgr.y (mgr.25723) 17 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.295458+0000 mgr.y (mgr.25723) 18 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.510173+0000 mgr.y (mgr.25723) 19 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.510573+0000 mgr.y (mgr.25723) 20 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.676738+0000 mgr.y (mgr.25723) 21 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.677092+0000 mgr.y (mgr.25723) 22 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.842079+0000 mgr.y (mgr.25723) 23 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:08:35.892074+0000 mgr.y (mgr.25723) 24 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:09:32.773737+0000 mgr.y (mgr.25723) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:09:40.358315+0000 mgr.y (mgr.25723) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:09:42.777157+0000 mgr.y (mgr.25723) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:09:42.793877+0000 mgr.y (mgr.25723) 63 : cephadm 1 Deploying daemon mon.c on smithi125 2023-10-09T06:11:27.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:45.304586+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:45.304778+0000 mon.a (mon.0) 51 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm 1 Deploying daemon mgr.x on smithi155 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm 1 Deploying daemon mon.a on smithi125 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:19.000889+0000 mgr.y (mgr.25723) 1 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTING 2023-10-09T06:11:27.645 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:19.131789+0000 mgr.y (mgr.25723) 2 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:19.233938+0000 mgr.y (mgr.25723) 3 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:19.234253+0000 mgr.y (mgr.25723) 4 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTED 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:24.156040+0000 mgr.y (mgr.25723) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.295064+0000 mgr.y (mgr.25723) 17 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.295458+0000 mgr.y (mgr.25723) 18 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.510173+0000 mgr.y (mgr.25723) 19 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.510573+0000 mgr.y (mgr.25723) 20 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.676738+0000 mgr.y (mgr.25723) 21 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.677092+0000 mgr.y (mgr.25723) 22 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.842079+0000 mgr.y (mgr.25723) 23 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.646 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:08:35.892074+0000 mgr.y (mgr.25723) 24 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:09:32.773737+0000 mgr.y (mgr.25723) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:09:40.358315+0000 mgr.y (mgr.25723) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:09:42.777157+0000 mgr.y (mgr.25723) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:09:42.793877+0000 mgr.y (mgr.25723) 63 : cephadm 1 Deploying daemon mon.c on smithi125 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:45.304586+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:45.304778+0000 mon.a (mon.0) 51 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:46.546664+0000 mon.a (mon.0) 52 : audit 0 from='client.? 172.21.15.125:0/137748366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.647 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:48.548632+0000 mon.a (mon.0) 54 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:53.031672+0000 mon.a (mon.0) 64 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:53.038812+0000 mon.a (mon.0) 65 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:53.084121+0000 mon.a (mon.0) 66 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:53.415664+0000 mon.a (mon.0) 67 : audit 0 from='client.? 172.21.15.125:0/3443539135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:54.088249+0000 mon.a (mon.0) 73 : cluster 0 mgrmap e36: y(active, since 95s), standbys: x 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:54.088821+0000 mon.a (mon.0) 74 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:54.088883+0000 mon.a (mon.0) 75 : cluster 1 Cluster is now healthy 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:54.096032+0000 mon.a (mon.0) 76 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:54.469847+0000 mgr.y (mgr.25723) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:55.866200+0000 mon.a (mon.0) 77 : audit 0 from='client.? 172.21.15.125:0/2363395131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:56.470653+0000 mgr.y (mgr.25723) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:09:58.471021+0000 mgr.y (mgr.25723) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:59.026393+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:59.036703+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:59.887082+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:09:59.896749+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:00.000164+0000 mon.a (mon.0) 82 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:00.297843+0000 mgr.y (mgr.25723) 72 : audit 0 from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:00.472118+0000 mgr.y (mgr.25723) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.649 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:02.472762+0000 mgr.y (mgr.25723) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:46.546664+0000 mon.a (mon.0) 52 : audit 0 from='client.? 172.21.15.125:0/137748366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:48.548632+0000 mon.a (mon.0) 54 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:53.031672+0000 mon.a (mon.0) 64 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:53.038812+0000 mon.a (mon.0) 65 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:53.084121+0000 mon.a (mon.0) 66 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:53.415664+0000 mon.a (mon.0) 67 : audit 0 from='client.? 172.21.15.125:0/3443539135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:54.088249+0000 mon.a (mon.0) 73 : cluster 0 mgrmap e36: y(active, since 95s), standbys: x 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:54.088821+0000 mon.a (mon.0) 74 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:54.088883+0000 mon.a (mon.0) 75 : cluster 1 Cluster is now healthy 2023-10-09T06:11:27.650 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:54.096032+0000 mon.a (mon.0) 76 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:54.469847+0000 mgr.y (mgr.25723) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:55.866200+0000 mon.a (mon.0) 77 : audit 0 from='client.? 172.21.15.125:0/2363395131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:56.470653+0000 mgr.y (mgr.25723) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:09:58.471021+0000 mgr.y (mgr.25723) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:59.026393+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:59.036703+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:59.887082+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:09:59.896749+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:00.000164+0000 mon.a (mon.0) 82 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:00.297843+0000 mgr.y (mgr.25723) 72 : audit 0 from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:00.472118+0000 mgr.y (mgr.25723) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:02.472762+0000 mgr.y (mgr.25723) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:03.549109+0000 mon.a (mon.0) 83 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:04.473860+0000 mgr.y (mgr.25723) 75 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:04.680643+0000 mgr.y (mgr.25723) 76 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:10:04.681237+0000 mgr.y (mgr.25723) 77 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:04.687417+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:06.474546+0000 mgr.y (mgr.25723) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:08.475164+0000 mgr.y (mgr.25723) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.652 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:10.476093+0000 mgr.y (mgr.25723) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.548108+0000 mon.a (mon.0) 85 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.556169+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.557954+0000 mon.a (mon.0) 87 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.559176+0000 mon.a (mon.0) 88 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.565280+0000 mon.a (mon.0) 89 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.648236+0000 mon.a (mon.0) 90 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.651526+0000 mon.a (mon.0) 91 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.652853+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:11.663305+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:12.476689+0000 mgr.y (mgr.25723) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:14.477690+0000 mgr.y (mgr.25723) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.653 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:16.478368+0000 mgr.y (mgr.25723) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:03.549109+0000 mon.a (mon.0) 83 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:04.473860+0000 mgr.y (mgr.25723) 75 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:04.680643+0000 mgr.y (mgr.25723) 76 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:10:04.681237+0000 mgr.y (mgr.25723) 77 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:04.687417+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:06.474546+0000 mgr.y (mgr.25723) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:08.475164+0000 mgr.y (mgr.25723) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:10.476093+0000 mgr.y (mgr.25723) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.548108+0000 mon.a (mon.0) 85 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.556169+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.557954+0000 mon.a (mon.0) 87 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.559176+0000 mon.a (mon.0) 88 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.565280+0000 mon.a (mon.0) 89 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.648236+0000 mon.a (mon.0) 90 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.651526+0000 mon.a (mon.0) 91 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.652853+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:11.663305+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:12.476689+0000 mgr.y (mgr.25723) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:14.477690+0000 mgr.y (mgr.25723) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:16.478368+0000 mgr.y (mgr.25723) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:18.479096+0000 mgr.y (mgr.25723) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:18.549438+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:20.480190+0000 mgr.y (mgr.25723) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:22.480757+0000 mgr.y (mgr.25723) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:24.481832+0000 mgr.y (mgr.25723) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:26.482481+0000 mgr.y (mgr.25723) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:28.483091+0000 mgr.y (mgr.25723) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:30.484109+0000 mgr.y (mgr.25723) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:32.484717+0000 mgr.y (mgr.25723) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:33.551982+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:34.485685+0000 mgr.y (mgr.25723) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:36.486345+0000 mgr.y (mgr.25723) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:38.486980+0000 mgr.y (mgr.25723) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:40.488111+0000 mgr.y (mgr.25723) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:42.488823+0000 mgr.y (mgr.25723) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:44.489924+0000 mgr.y (mgr.25723) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:46.490619+0000 mgr.y (mgr.25723) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:48.491261+0000 mgr.y (mgr.25723) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:10:48.550271+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:50.492405+0000 mgr.y (mgr.25723) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:18.479096+0000 mgr.y (mgr.25723) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:18.549438+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:20.480190+0000 mgr.y (mgr.25723) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:22.480757+0000 mgr.y (mgr.25723) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:24.481832+0000 mgr.y (mgr.25723) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:26.482481+0000 mgr.y (mgr.25723) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:28.483091+0000 mgr.y (mgr.25723) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:30.484109+0000 mgr.y (mgr.25723) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:32.484717+0000 mgr.y (mgr.25723) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:33.551982+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:34.485685+0000 mgr.y (mgr.25723) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:36.486345+0000 mgr.y (mgr.25723) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:38.486980+0000 mgr.y (mgr.25723) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:40.488111+0000 mgr.y (mgr.25723) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:42.488823+0000 mgr.y (mgr.25723) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:44.489924+0000 mgr.y (mgr.25723) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:46.490619+0000 mgr.y (mgr.25723) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:48.491261+0000 mgr.y (mgr.25723) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:10:48.550271+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.659 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:50.492405+0000 mgr.y (mgr.25723) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:52.493110+0000 mgr.y (mgr.25723) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:54.494198+0000 mgr.y (mgr.25723) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:56.494914+0000 mgr.y (mgr.25723) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:10:58.495690+0000 mgr.y (mgr.25723) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:00.496663+0000 mgr.y (mgr.25723) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:02.497266+0000 mgr.y (mgr.25723) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:03.550630+0000 mon.a (mon.0) 97 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:04.498469+0000 mgr.y (mgr.25723) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:06.499114+0000 mgr.y (mgr.25723) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:08.499785+0000 mgr.y (mgr.25723) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:10.500842+0000 mgr.y (mgr.25723) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:11.747260+0000 mon.a (mon.0) 98 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:12.501490+0000 mgr.y (mgr.25723) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:14.502626+0000 mgr.y (mgr.25723) 112 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:15.176977+0000 mgr.y (mgr.25723) 113 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:11:15.177588+0000 mgr.y (mgr.25723) 114 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:15.183795+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:16.503218+0000 mgr.y (mgr.25723) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:17.745083+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:17.754642+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.044758+0000 mon.a (mon.0) 102 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:52.493110+0000 mgr.y (mgr.25723) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:54.494198+0000 mgr.y (mgr.25723) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:56.494914+0000 mgr.y (mgr.25723) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:10:58.495690+0000 mgr.y (mgr.25723) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:00.496663+0000 mgr.y (mgr.25723) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:02.497266+0000 mgr.y (mgr.25723) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:03.550630+0000 mon.a (mon.0) 97 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:04.498469+0000 mgr.y (mgr.25723) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:06.499114+0000 mgr.y (mgr.25723) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:08.499785+0000 mgr.y (mgr.25723) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:10.500842+0000 mgr.y (mgr.25723) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:11.747260+0000 mon.a (mon.0) 98 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:12.501490+0000 mgr.y (mgr.25723) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:14.502626+0000 mgr.y (mgr.25723) 112 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:15.176977+0000 mgr.y (mgr.25723) 113 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:11:15.177588+0000 mgr.y (mgr.25723) 114 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:15.183795+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.663 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:16.503218+0000 mgr.y (mgr.25723) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:17.745083+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:17.754642+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.044758+0000 mon.a (mon.0) 102 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.050522+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:18.503762+0000 mgr.y (mgr.25723) 116 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.520925+0000 mon.a (mon.0) 104 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.522613+0000 mon.a (mon.0) 105 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.530032+0000 mon.a (mon.0) 106 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.550142+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.641619+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.645863+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:27.664 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.647911+0000 mon.a (mon.0) 110 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:18.649198+0000 mon.a (mon.0) 111 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:11:18.650022+0000 mgr.y (mgr.25723) 117 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:20.504758+0000 mgr.y (mgr.25723) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:11:21.320724+0000 mgr.y (mgr.25723) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:21.328636+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:21.329960+0000 mon.a (mon.0) 113 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:21.330972+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:21.331918+0000 mon.a (mon.0) 115 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cephadm 2023-10-09T06:11:21.333142+0000 mgr.y (mgr.25723) 120 : cephadm 1 Deploying daemon mon.b on smithi155 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:22.505338+0000 mgr.y (mgr.25723) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:23.497762+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.125:0/2832843321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.665 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.239253+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.050522+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:18.503762+0000 mgr.y (mgr.25723) 116 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.520925+0000 mon.a (mon.0) 104 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.522613+0000 mon.a (mon.0) 105 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.530032+0000 mon.a (mon.0) 106 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.550142+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.641619+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.645863+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:27.666 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.647911+0000 mon.a (mon.0) 110 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:18.649198+0000 mon.a (mon.0) 111 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:11:18.650022+0000 mgr.y (mgr.25723) 117 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:20.504758+0000 mgr.y (mgr.25723) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:11:21.320724+0000 mgr.y (mgr.25723) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:21.328636+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:21.329960+0000 mon.a (mon.0) 113 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:21.330972+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:21.331918+0000 mon.a (mon.0) 115 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cephadm 2023-10-09T06:11:21.333142+0000 mgr.y (mgr.25723) 120 : cephadm 1 Deploying daemon mon.b on smithi155 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:22.505338+0000 mgr.y (mgr.25723) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:23.497762+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.125:0/2832843321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.239253+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.239690+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.240090+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.240134+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.241661+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.241868+0000 mon.a (mon.0) 126 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.242066+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.262680+0000 mon.a (mon.0) 127 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:11:27.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.273577+0000 mon.a (mon.0) 128 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.274437+0000 mon.a (mon.0) 129 : cluster 0 fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.274527+0000 mon.a (mon.0) 130 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.274948+0000 mon.a (mon.0) 131 : cluster 0 mgrmap e36: y(active, since 3m), standbys: x 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.292722+0000 mon.a (mon.0) 132 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.300024+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.308216+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: audit 2023-10-09T06:11:26.358408+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[80002]: cluster 2023-10-09T06:11:26.507006+0000 mgr.y (mgr.25723) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.669 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.239690+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.240090+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.240134+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.241661+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.241868+0000 mon.a (mon.0) 126 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.242066+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.262680+0000 mon.a (mon.0) 127 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.273577+0000 mon.a (mon.0) 128 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.274437+0000 mon.a (mon.0) 129 : cluster 0 fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.274527+0000 mon.a (mon.0) 130 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.274948+0000 mon.a (mon.0) 131 : cluster 0 mgrmap e36: y(active, since 3m), standbys: x 2023-10-09T06:11:27.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.292722+0000 mon.a (mon.0) 132 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.671 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.300024+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.671 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.308216+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.671 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: audit 2023-10-09T06:11:26.358408+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.671 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:27 smithi125 bash[77299]: cluster 2023-10-09T06:11:26.507006+0000 mgr.y (mgr.25723) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T05:50:59.273186+0000 mon.a (mon.0) 0 : 1 mkfs 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:25.085692+0000 mgr.x (mgr.25270) 47 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:25.086168+0000 mgr.x (mgr.25270) 48 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi155 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:26.904400+0000 mgr.x (mgr.25270) 50 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:26.922727+0000 mgr.x (mgr.25270) 51 : cephadm 1 Reconfiguring daemon grafana.a on smithi155 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:52.066435+0000 mgr.x (mgr.25270) 67 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:52.067027+0000 mgr.x (mgr.25270) 68 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:11:27.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:54.487439+0000 mgr.x (mgr.25270) 71 : cephadm 1 Upgrade: Updating mgr.y 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:06:54.505217+0000 mgr.x (mgr.25270) 72 : cephadm 1 Deploying daemon mgr.y on smithi125 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:15.803578+0000 mgr.y (mgr.15930) 2 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Bus STARTING 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:15.925540+0000 mgr.y (mgr.15930) 3 : cephadm 1 [09/Oct/2023:06:07:15] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:16.027676+0000 mgr.y (mgr.15930) 4 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:16.027989+0000 mgr.y (mgr.15930) 5 : cephadm 1 [09/Oct/2023:06:07:16] ENGINE Bus STARTED 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.474466+0000 mgr.y (mgr.15930) 14 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.474797+0000 mgr.y (mgr.15930) 15 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.639409+0000 mgr.y (mgr.15930) 16 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.640236+0000 mgr.y (mgr.15930) 17 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.804714+0000 mgr.y (mgr.15930) 18 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.807013+0000 mgr.y (mgr.15930) 19 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.673 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.969268+0000 mgr.y (mgr.15930) 20 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:31.973131+0000 mgr.y (mgr.15930) 21 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:32.251621+0000 mgr.y (mgr.15930) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:32.515120+0000 mgr.y (mgr.15930) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi155 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:40.588750+0000 mgr.y (mgr.15930) 29 : cephadm 1 Upgrade: Updating mgr.x 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:07:40.595845+0000 mgr.y (mgr.15930) 30 : cephadm 1 Deploying daemon mgr.x on smithi155 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:00.965813+0000 mgr.y (mgr.15930) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:00.993912+0000 mgr.y (mgr.15930) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:03.463417+0000 mgr.y (mgr.15930) 45 : cephadm 1 Upgrade: Updating mon.a 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:03.477928+0000 mgr.y (mgr.15930) 46 : cephadm 1 Deploying daemon mon.a on smithi125 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:19.000889+0000 mgr.y (mgr.25723) 1 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTING 2023-10-09T06:11:27.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:19.131789+0000 mgr.y (mgr.25723) 2 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on https://172.21.15.125:7150 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:19.233938+0000 mgr.y (mgr.25723) 3 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Serving on http://172.21.15.125:8765 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:19.234253+0000 mgr.y (mgr.25723) 4 : cephadm 1 [09/Oct/2023:06:08:19] ENGINE Bus STARTED 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:24.156040+0000 mgr.y (mgr.25723) 10 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.295064+0000 mgr.y (mgr.25723) 17 : cephadm 1 Updating smithi125:/etc/ceph/ceph.conf 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.295458+0000 mgr.y (mgr.25723) 18 : cephadm 1 Updating smithi155:/etc/ceph/ceph.conf 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.510173+0000 mgr.y (mgr.25723) 19 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.510573+0000 mgr.y (mgr.25723) 20 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.676738+0000 mgr.y (mgr.25723) 21 : cephadm 1 Updating smithi125:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.677092+0000 mgr.y (mgr.25723) 22 : cephadm 1 Updating smithi155:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.842079+0000 mgr.y (mgr.25723) 23 : cephadm 1 Updating smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:08:35.892074+0000 mgr.y (mgr.25723) 24 : cephadm 1 Updating smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:11:27.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:09:32.773737+0000 mgr.y (mgr.25723) 55 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:09:40.358315+0000 mgr.y (mgr.25723) 59 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:09:42.777157+0000 mgr.y (mgr.25723) 62 : cephadm 1 Upgrade: Updating mon.c 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:09:42.793877+0000 mgr.y (mgr.25723) 63 : cephadm 1 Deploying daemon mon.c on smithi125 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:45.304586+0000 mon.a (mon.0) 50 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:45.304778+0000 mon.a (mon.0) 51 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:46.546664+0000 mon.a (mon.0) 52 : audit 0 from='client.? 172.21.15.125:0/137748366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:48.548632+0000 mon.a (mon.0) 54 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:53.031672+0000 mon.a (mon.0) 64 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:53.038812+0000 mon.a (mon.0) 65 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:53.084121+0000 mon.a (mon.0) 66 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:53.415664+0000 mon.a (mon.0) 67 : audit 0 from='client.? 172.21.15.125:0/3443539135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:54.088249+0000 mon.a (mon.0) 73 : cluster 0 mgrmap e36: y(active, since 95s), standbys: x 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:54.088821+0000 mon.a (mon.0) 74 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:54.088883+0000 mon.a (mon.0) 75 : cluster 1 Cluster is now healthy 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:54.096032+0000 mon.a (mon.0) 76 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:54.469847+0000 mgr.y (mgr.25723) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:55.866200+0000 mon.a (mon.0) 77 : audit 0 from='client.? 172.21.15.125:0/2363395131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:56.470653+0000 mgr.y (mgr.25723) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:09:58.471021+0000 mgr.y (mgr.25723) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:59.026393+0000 mon.a (mon.0) 78 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:59.036703+0000 mon.a (mon.0) 79 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:59.887082+0000 mon.a (mon.0) 80 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:09:59.896749+0000 mon.a (mon.0) 81 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:00.000164+0000 mon.a (mon.0) 82 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:00.297843+0000 mgr.y (mgr.25723) 72 : audit 0 from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:00.472118+0000 mgr.y (mgr.25723) 73 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:02.472762+0000 mgr.y (mgr.25723) 74 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:03.549109+0000 mon.a (mon.0) 83 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:04.473860+0000 mgr.y (mgr.25723) 75 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:04.680643+0000 mgr.y (mgr.25723) 76 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:10:04.681237+0000 mgr.y (mgr.25723) 77 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:04.687417+0000 mon.a (mon.0) 84 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:06.474546+0000 mgr.y (mgr.25723) 78 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:08.475164+0000 mgr.y (mgr.25723) 79 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:10.476093+0000 mgr.y (mgr.25723) 80 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.548108+0000 mon.a (mon.0) 85 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.556169+0000 mon.a (mon.0) 86 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.557954+0000 mon.a (mon.0) 87 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.559176+0000 mon.a (mon.0) 88 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.565280+0000 mon.a (mon.0) 89 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.648236+0000 mon.a (mon.0) 90 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.651526+0000 mon.a (mon.0) 91 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.652853+0000 mon.a (mon.0) 92 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:11.663305+0000 mon.a (mon.0) 93 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:12.476689+0000 mgr.y (mgr.25723) 81 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.679 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:14.477690+0000 mgr.y (mgr.25723) 82 : cluster 0 pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:16.478368+0000 mgr.y (mgr.25723) 83 : cluster 0 pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:18.479096+0000 mgr.y (mgr.25723) 84 : cluster 0 pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:18.549438+0000 mon.a (mon.0) 94 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:20.480190+0000 mgr.y (mgr.25723) 85 : cluster 0 pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:22.480757+0000 mgr.y (mgr.25723) 86 : cluster 0 pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:24.481832+0000 mgr.y (mgr.25723) 87 : cluster 0 pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:26.482481+0000 mgr.y (mgr.25723) 88 : cluster 0 pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:28.483091+0000 mgr.y (mgr.25723) 89 : cluster 0 pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:30.484109+0000 mgr.y (mgr.25723) 90 : cluster 0 pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:32.484717+0000 mgr.y (mgr.25723) 91 : cluster 0 pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:33.551982+0000 mon.a (mon.0) 95 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.680 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:34.485685+0000 mgr.y (mgr.25723) 92 : cluster 0 pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:36.486345+0000 mgr.y (mgr.25723) 93 : cluster 0 pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:38.486980+0000 mgr.y (mgr.25723) 94 : cluster 0 pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:40.488111+0000 mgr.y (mgr.25723) 95 : cluster 0 pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:42.488823+0000 mgr.y (mgr.25723) 96 : cluster 0 pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:44.489924+0000 mgr.y (mgr.25723) 97 : cluster 0 pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:46.490619+0000 mgr.y (mgr.25723) 98 : cluster 0 pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:48.491261+0000 mgr.y (mgr.25723) 99 : cluster 0 pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:10:48.550271+0000 mon.a (mon.0) 96 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:50.492405+0000 mgr.y (mgr.25723) 100 : cluster 0 pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:52.493110+0000 mgr.y (mgr.25723) 101 : cluster 0 pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:54.494198+0000 mgr.y (mgr.25723) 102 : cluster 0 pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:56.494914+0000 mgr.y (mgr.25723) 103 : cluster 0 pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:10:58.495690+0000 mgr.y (mgr.25723) 104 : cluster 0 pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:00.496663+0000 mgr.y (mgr.25723) 105 : cluster 0 pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:02.497266+0000 mgr.y (mgr.25723) 106 : cluster 0 pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:03.550630+0000 mon.a (mon.0) 97 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:04.498469+0000 mgr.y (mgr.25723) 107 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:06.499114+0000 mgr.y (mgr.25723) 108 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:08.499785+0000 mgr.y (mgr.25723) 109 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:10.500842+0000 mgr.y (mgr.25723) 110 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:11.747260+0000 mon.a (mon.0) 98 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:12.501490+0000 mgr.y (mgr.25723) 111 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:14.502626+0000 mgr.y (mgr.25723) 112 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:15.176977+0000 mgr.y (mgr.25723) 113 : audit 0 from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:11:15.177588+0000 mgr.y (mgr.25723) 114 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:15.183795+0000 mon.a (mon.0) 99 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:16.503218+0000 mgr.y (mgr.25723) 115 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:17.745083+0000 mon.a (mon.0) 100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:17.754642+0000 mon.a (mon.0) 101 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.044758+0000 mon.a (mon.0) 102 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.050522+0000 mon.a (mon.0) 103 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:18.503762+0000 mgr.y (mgr.25723) 116 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.520925+0000 mon.a (mon.0) 104 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.522613+0000 mon.a (mon.0) 105 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.530032+0000 mon.a (mon.0) 106 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.550142+0000 mon.a (mon.0) 107 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.641619+0000 mon.a (mon.0) 108 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.645863+0000 mon.a (mon.0) 109 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.647911+0000 mon.a (mon.0) 110 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:18.649198+0000 mon.a (mon.0) 111 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:11:18.650022+0000 mgr.y (mgr.25723) 117 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:20.504758+0000 mgr.y (mgr.25723) 118 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:11:21.320724+0000 mgr.y (mgr.25723) 119 : cephadm 1 Upgrade: Updating mon.b 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:21.328636+0000 mon.a (mon.0) 112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:21.329960+0000 mon.a (mon.0) 113 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:21.330972+0000 mon.a (mon.0) 114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:27.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:21.331918+0000 mon.a (mon.0) 115 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cephadm 2023-10-09T06:11:21.333142+0000 mgr.y (mgr.25723) 120 : cephadm 1 Deploying daemon mon.b on smithi155 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:22.505338+0000 mgr.y (mgr.25723) 121 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:23.497762+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.125:0/2832843321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.239253+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.239690+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.240090+0000 mon.a (mon.0) 124 : cluster 1 mon.a calling monitor election 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.240134+0000 mon.c (mon.2) 4 : cluster 1 mon.c calling monitor election 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.241661+0000 mon.a (mon.0) 125 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.241868+0000 mon.a (mon.0) 126 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.242066+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-09T06:11:27.685 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.262680+0000 mon.a (mon.0) 127 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.273577+0000 mon.a (mon.0) 128 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0],b=[v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0],c=[v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]} removed_ranks: {} 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.274437+0000 mon.a (mon.0) 129 : cluster 0 fsmap foo:1 {0=foo.smithi125.yetyhy=up:active} 1 up:standby 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.274527+0000 mon.a (mon.0) 130 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.274948+0000 mon.a (mon.0) 131 : cluster 0 mgrmap e36: y(active, since 3m), standbys: x 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.292722+0000 mon.a (mon.0) 132 : cluster 1 overall HEALTH_OK 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.300024+0000 mon.a (mon.0) 133 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.308216+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: audit 2023-10-09T06:11:26.358408+0000 mon.a (mon.0) 135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:27.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:27 smithi155 bash[59495]: cluster 2023-10-09T06:11:26.507006+0000 mgr.y (mgr.25723) 123 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:28.407 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T06:11:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:28 smithi125 bash[77299]: audit 2023-10-09T06:11:28.387392+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.125:0/1214553032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:28.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:28 smithi125 bash[80002]: audit 2023-10-09T06:11:28.387392+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.125:0/1214553032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:28 smithi155 bash[59495]: audit 2023-10-09T06:11:28.387392+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.125:0/1214553032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:11:29.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:11:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:29 smithi125 bash[77299]: cluster 2023-10-09T06:11:28.507627+0000 mgr.y (mgr.25723) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:29.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:29 smithi125 bash[80002]: cluster 2023-10-09T06:11:28.507627+0000 mgr.y (mgr.25723) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:29 smithi155 bash[59495]: cluster 2023-10-09T06:11:28.507627+0000 mgr.y (mgr.25723) 124 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:31.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:31 smithi125 bash[73481]: debug 2023-10-09T06:11:31.033+0000 7f8497396700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T06:11:31.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:31] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.33.4" 2023-10-09T06:11:31.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:31 smithi125 bash[77299]: cluster 2023-10-09T06:11:30.508859+0000 mgr.y (mgr.25723) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:31 smithi125 bash[80002]: cluster 2023-10-09T06:11:30.508859+0000 mgr.y (mgr.25723) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:31 smithi155 bash[59495]: cluster 2023-10-09T06:11:30.508859+0000 mgr.y (mgr.25723) 125 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (5m) - 11m 13.8M - 0.23.0 ba2b418f427c 287b260dd8a4 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (4m) - 11m 40.4M - 8.3.5 dad864ee21e9 988e098b0939 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (9m) - 9m 16.3M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (9m) - 9m 13.9M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (3m) - 18m 427M - 18.2.0-598-gb4fa663f 3060897fdf97 6a90d5aaacfd 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (4m) - 20m 475M - 18.2.0-598-gb4fa663f 3060897fdf97 4731d4d85533 2023-10-09T06:11:32.944 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (3m) - 20m 57.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 66d70cab6c5d 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (7s) - 18m 19.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 1d811296fe1c 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (106s) - 18m 36.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 bedca0a9cc9d 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (5m) - 11m 9287k - 1.3.1 1dbe0e931976 951da6650ff3 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (5m) - 11m 9459k - 1.3.1 1dbe0e931976 4e060b6a5bef 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (17m) - 17m 46.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4276c724754b 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (17m) - 17m 46.4M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 67dcf3157f53 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (16m) - 16m 43.9M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 05b5c73536d5 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (15m) - 15m 43.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 6d440ba42e37 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (14m) - 15m 43.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (14m) - 14m 44.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (13m) - 13m 44.8M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:11:32.945 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (12m) - 12m 48.5M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:11:32.946 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (3m) - 11m 56.8M - 2.33.4 514e6a882f6e 5a577340c8e2 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: cluster 2023-10-09T06:11:32.509570+0000 mgr.y (mgr.25723) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:32.608340+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:32.617928+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:32.923341+0000 mgr.y (mgr.25723) 127 : audit 0 from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:33.417020+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:33.425742+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[80002]: audit 2023-10-09T06:11:33.551235+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:33.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: cluster 2023-10-09T06:11:32.509570+0000 mgr.y (mgr.25723) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:32.608340+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:32.617928+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:32.923341+0000 mgr.y (mgr.25723) 127 : audit 0 from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:33.417020+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:33.425742+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:33 smithi125 bash[77299]: audit 2023-10-09T06:11:33.551235+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: cluster 2023-10-09T06:11:32.509570+0000 mgr.y (mgr.25723) 126 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:32.608340+0000 mon.a (mon.0) 136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:32.617928+0000 mon.a (mon.0) 137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:32.923341+0000 mgr.y (mgr.25723) 127 : audit 0 from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:33.417020+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:33.425742+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:33 smithi155 bash[59495]: audit 2023-10-09T06:11:33.551235+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:35.864 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:35 smithi125 bash[77299]: cluster 2023-10-09T06:11:34.510656+0000 mgr.y (mgr.25723) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:35.864 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:35 smithi125 bash[80002]: cluster 2023-10-09T06:11:34.510656+0000 mgr.y (mgr.25723) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:35 smithi155 bash[59495]: cluster 2023-10-09T06:11:34.510656+0000 mgr.y (mgr.25723) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:37.280 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:37 smithi125 bash[77299]: cluster 2023-10-09T06:11:36.511275+0000 mgr.y (mgr.25723) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:37 smithi125 bash[77299]: audit 2023-10-09T06:11:37.277292+0000 mon.a (mon.0) 141 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:37 smithi125 bash[80002]: cluster 2023-10-09T06:11:36.511275+0000 mgr.y (mgr.25723) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:37 smithi125 bash[80002]: audit 2023-10-09T06:11:37.277292+0000 mon.a (mon.0) 141 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:37 smithi155 bash[59495]: cluster 2023-10-09T06:11:36.511275+0000 mgr.y (mgr.25723) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:37 smithi155 bash[59495]: audit 2023-10-09T06:11:37.277292+0000 mon.a (mon.0) 141 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:38.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:11:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:38 smithi125 bash[80002]: audit 2023-10-09T06:11:37.269904+0000 mgr.y (mgr.25723) 130 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:38 smithi125 bash[80002]: cephadm 2023-10-09T06:11:37.270537+0000 mgr.y (mgr.25723) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:38 smithi125 bash[77299]: audit 2023-10-09T06:11:37.269904+0000 mgr.y (mgr.25723) 130 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:38 smithi125 bash[77299]: cephadm 2023-10-09T06:11:37.270537+0000 mgr.y (mgr.25723) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:38 smithi155 bash[59495]: audit 2023-10-09T06:11:37.269904+0000 mgr.y (mgr.25723) 130 : audit 0 from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:11:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:38 smithi155 bash[59495]: cephadm 2023-10-09T06:11:37.270537+0000 mgr.y (mgr.25723) 131 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:11:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:39 smithi155 bash[59495]: cluster 2023-10-09T06:11:38.511941+0000 mgr.y (mgr.25723) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:39.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:39 smithi125 bash[77299]: cluster 2023-10-09T06:11:38.511941+0000 mgr.y (mgr.25723) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:39.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:39 smithi125 bash[80002]: cluster 2023-10-09T06:11:38.511941+0000 mgr.y (mgr.25723) 132 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:41.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:41] "GET /metrics HTTP/1.1" 200 33921 "" "Prometheus/2.33.4" 2023-10-09T06:11:41.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:41 smithi125 bash[77299]: cluster 2023-10-09T06:11:40.512981+0000 mgr.y (mgr.25723) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:41 smithi125 bash[80002]: cluster 2023-10-09T06:11:40.512981+0000 mgr.y (mgr.25723) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:41.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:41 smithi155 bash[59495]: cluster 2023-10-09T06:11:40.512981+0000 mgr.y (mgr.25723) 133 : cluster 0 pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: cluster 2023-10-09T06:11:42.513540+0000 mgr.y (mgr.25723) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.174144+0000 mon.a (mon.0) 142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.182021+0000 mon.a (mon.0) 143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.183729+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.185133+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.192065+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.263800+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.265601+0000 mon.a (mon.0) 148 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[77299]: audit 2023-10-09T06:11:43.267293+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: cluster 2023-10-09T06:11:42.513540+0000 mgr.y (mgr.25723) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.174144+0000 mon.a (mon.0) 142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.182021+0000 mon.a (mon.0) 143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.183729+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.185133+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.192065+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.263800+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.265601+0000 mon.a (mon.0) 148 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:43 smithi125 bash[80002]: audit 2023-10-09T06:11:43.267293+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: cluster 2023-10-09T06:11:42.513540+0000 mgr.y (mgr.25723) 134 : cluster 0 pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.174144+0000 mon.a (mon.0) 142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.182021+0000 mon.a (mon.0) 143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.183729+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.185133+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.192065+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:43.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.263800+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:43.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.265601+0000 mon.a (mon.0) 148 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:43.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:43 smithi155 bash[59495]: audit 2023-10-09T06:11:43.267293+0000 mon.a (mon.0) 149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:44 smithi155 bash[59495]: cephadm 2023-10-09T06:11:43.263166+0000 mgr.y (mgr.25723) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T06:11:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:44 smithi155 bash[59495]: cephadm 2023-10-09T06:11:43.269092+0000 mgr.y (mgr.25723) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi125 2023-10-09T06:11:44.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:44 smithi125 bash[77299]: cephadm 2023-10-09T06:11:43.263166+0000 mgr.y (mgr.25723) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T06:11:44.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:44 smithi125 bash[77299]: cephadm 2023-10-09T06:11:43.269092+0000 mgr.y (mgr.25723) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi125 2023-10-09T06:11:44.988 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:44 smithi125 bash[80002]: cephadm 2023-10-09T06:11:43.263166+0000 mgr.y (mgr.25723) 135 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-09T06:11:44.988 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:44 smithi125 bash[80002]: cephadm 2023-10-09T06:11:43.269092+0000 mgr.y (mgr.25723) 136 : cephadm 1 Reconfiguring daemon mon.a on smithi125 2023-10-09T06:11:46.091 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:45 smithi125 bash[77299]: cluster 2023-10-09T06:11:44.514265+0000 mgr.y (mgr.25723) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:46.091 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:45 smithi125 bash[80002]: cluster 2023-10-09T06:11:44.514265+0000 mgr.y (mgr.25723) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:45 smithi155 bash[59495]: cluster 2023-10-09T06:11:44.514265+0000 mgr.y (mgr.25723) 137 : cluster 0 pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:47.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: audit 2023-10-09T06:11:45.736866+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: audit 2023-10-09T06:11:45.746404+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: cephadm 2023-10-09T06:11:45.789746+0000 mgr.y (mgr.25723) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:11:47.018 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: audit 2023-10-09T06:11:45.790611+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: audit 2023-10-09T06:11:45.792840+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: audit 2023-10-09T06:11:45.794894+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[77299]: cephadm 2023-10-09T06:11:45.796918+0000 mgr.y (mgr.25723) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi125 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: audit 2023-10-09T06:11:45.736866+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: audit 2023-10-09T06:11:45.746404+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.019 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: cephadm 2023-10-09T06:11:45.789746+0000 mgr.y (mgr.25723) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:11:47.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: audit 2023-10-09T06:11:45.790611+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:11:47.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: audit 2023-10-09T06:11:45.792840+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:11:47.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: audit 2023-10-09T06:11:45.794894+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:47.020 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:46 smithi125 bash[80002]: cephadm 2023-10-09T06:11:45.796918+0000 mgr.y (mgr.25723) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi125 2023-10-09T06:11:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: audit 2023-10-09T06:11:45.736866+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: audit 2023-10-09T06:11:45.746404+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: cephadm 2023-10-09T06:11:45.789746+0000 mgr.y (mgr.25723) 138 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:11:47.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: audit 2023-10-09T06:11:45.790611+0000 mon.a (mon.0) 152 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:11:47.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: audit 2023-10-09T06:11:45.792840+0000 mon.a (mon.0) 153 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:11:47.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: audit 2023-10-09T06:11:45.794894+0000 mon.a (mon.0) 154 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:47.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:46 smithi155 bash[59495]: cephadm 2023-10-09T06:11:45.796918+0000 mgr.y (mgr.25723) 139 : cephadm 1 Reconfiguring daemon mgr.y on smithi125 2023-10-09T06:11:48.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:47 smithi125 bash[77299]: cluster 2023-10-09T06:11:46.514870+0000 mgr.y (mgr.25723) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:48.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:47 smithi125 bash[80002]: cluster 2023-10-09T06:11:46.514870+0000 mgr.y (mgr.25723) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:48.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:47 smithi155 bash[59495]: cluster 2023-10-09T06:11:46.514870+0000 mgr.y (mgr.25723) 140 : cluster 0 pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:47.943287+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:47.952623+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: cephadm 2023-10-09T06:11:47.965686+0000 mgr.y (mgr.25723) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:47.966287+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:47.967892+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:47.969486+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:49.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[77299]: audit 2023-10-09T06:11:48.551595+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:47.943287+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:47.952623+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: cephadm 2023-10-09T06:11:47.965686+0000 mgr.y (mgr.25723) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:47.966287+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:47.967892+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:47.969486+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:49.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:48 smithi125 bash[80002]: audit 2023-10-09T06:11:48.551595+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:47.943287+0000 mon.a (mon.0) 155 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:47.952623+0000 mon.a (mon.0) 156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: cephadm 2023-10-09T06:11:47.965686+0000 mgr.y (mgr.25723) 141 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-09T06:11:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:47.966287+0000 mon.a (mon.0) 157 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:11:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:47.967892+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:11:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:47.969486+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:48 smithi155 bash[59495]: audit 2023-10-09T06:11:48.551595+0000 mon.a (mon.0) 160 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:50.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:49 smithi125 bash[77299]: cephadm 2023-10-09T06:11:47.971281+0000 mgr.y (mgr.25723) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi125 2023-10-09T06:11:50.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:49 smithi125 bash[77299]: cluster 2023-10-09T06:11:48.515456+0000 mgr.y (mgr.25723) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:50.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:49 smithi125 bash[80002]: cephadm 2023-10-09T06:11:47.971281+0000 mgr.y (mgr.25723) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi125 2023-10-09T06:11:50.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:49 smithi125 bash[80002]: cluster 2023-10-09T06:11:48.515456+0000 mgr.y (mgr.25723) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:49 smithi155 bash[59495]: cephadm 2023-10-09T06:11:47.971281+0000 mgr.y (mgr.25723) 142 : cephadm 1 Reconfiguring daemon mon.c on smithi125 2023-10-09T06:11:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:49 smithi155 bash[59495]: cluster 2023-10-09T06:11:48.515456+0000 mgr.y (mgr.25723) 143 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:51.498 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:11:51] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-10-09T06:11:51.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: audit 2023-10-09T06:11:50.146867+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: audit 2023-10-09T06:11:50.155040+0000 mon.a (mon.0) 162 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: cephadm 2023-10-09T06:11:50.190370+0000 mgr.y (mgr.25723) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: audit 2023-10-09T06:11:50.191031+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: audit 2023-10-09T06:11:50.192782+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: cephadm 2023-10-09T06:11:50.197152+0000 mgr.y (mgr.25723) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi125 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[77299]: cluster 2023-10-09T06:11:50.516557+0000 mgr.y (mgr.25723) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: audit 2023-10-09T06:11:50.146867+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: audit 2023-10-09T06:11:50.155040+0000 mon.a (mon.0) 162 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: cephadm 2023-10-09T06:11:50.190370+0000 mgr.y (mgr.25723) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: audit 2023-10-09T06:11:50.191031+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: audit 2023-10-09T06:11:50.192782+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:51.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: cephadm 2023-10-09T06:11:50.197152+0000 mgr.y (mgr.25723) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi125 2023-10-09T06:11:51.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:51 smithi125 bash[80002]: cluster 2023-10-09T06:11:50.516557+0000 mgr.y (mgr.25723) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: audit 2023-10-09T06:11:50.146867+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: audit 2023-10-09T06:11:50.155040+0000 mon.a (mon.0) 162 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: cephadm 2023-10-09T06:11:50.190370+0000 mgr.y (mgr.25723) 144 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:11:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: audit 2023-10-09T06:11:50.191031+0000 mon.a (mon.0) 163 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:11:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: audit 2023-10-09T06:11:50.192782+0000 mon.a (mon.0) 164 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:51.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: cephadm 2023-10-09T06:11:50.197152+0000 mgr.y (mgr.25723) 145 : cephadm 1 Reconfiguring daemon osd.0 on smithi125 2023-10-09T06:11:51.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:51 smithi155 bash[59495]: cluster 2023-10-09T06:11:50.516557+0000 mgr.y (mgr.25723) 146 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: cluster 2023-10-09T06:11:52.517209+0000 mgr.y (mgr.25723) 147 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: audit 2023-10-09T06:11:52.580264+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: audit 2023-10-09T06:11:52.590958+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: cephadm 2023-10-09T06:11:52.619272+0000 mgr.y (mgr.25723) 148 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: audit 2023-10-09T06:11:52.619931+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:11:53.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: audit 2023-10-09T06:11:52.621437+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[77299]: cephadm 2023-10-09T06:11:52.625279+0000 mgr.y (mgr.25723) 149 : cephadm 1 Reconfiguring daemon osd.1 on smithi125 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: cluster 2023-10-09T06:11:52.517209+0000 mgr.y (mgr.25723) 147 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: audit 2023-10-09T06:11:52.580264+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: audit 2023-10-09T06:11:52.590958+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: cephadm 2023-10-09T06:11:52.619272+0000 mgr.y (mgr.25723) 148 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: audit 2023-10-09T06:11:52.619931+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: audit 2023-10-09T06:11:52.621437+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:53.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:53 smithi125 bash[80002]: cephadm 2023-10-09T06:11:52.625279+0000 mgr.y (mgr.25723) 149 : cephadm 1 Reconfiguring daemon osd.1 on smithi125 2023-10-09T06:11:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: cluster 2023-10-09T06:11:52.517209+0000 mgr.y (mgr.25723) 147 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: audit 2023-10-09T06:11:52.580264+0000 mon.a (mon.0) 165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: audit 2023-10-09T06:11:52.590958+0000 mon.a (mon.0) 166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: cephadm 2023-10-09T06:11:52.619272+0000 mgr.y (mgr.25723) 148 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:11:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: audit 2023-10-09T06:11:52.619931+0000 mon.a (mon.0) 167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:11:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: audit 2023-10-09T06:11:52.621437+0000 mon.a (mon.0) 168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:53 smithi155 bash[59495]: cephadm 2023-10-09T06:11:52.625279+0000 mgr.y (mgr.25723) 149 : cephadm 1 Reconfiguring daemon osd.1 on smithi125 2023-10-09T06:11:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: cluster 2023-10-09T06:11:54.518136+0000 mgr.y (mgr.25723) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: audit 2023-10-09T06:11:54.853625+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: audit 2023-10-09T06:11:54.861397+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: cephadm 2023-10-09T06:11:54.894747+0000 mgr.y (mgr.25723) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:11:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: audit 2023-10-09T06:11:54.895232+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:11:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: audit 2023-10-09T06:11:54.896592+0000 mon.a (mon.0) 172 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[77299]: cephadm 2023-10-09T06:11:54.900545+0000 mgr.y (mgr.25723) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi125 2023-10-09T06:11:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: cluster 2023-10-09T06:11:54.518136+0000 mgr.y (mgr.25723) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: audit 2023-10-09T06:11:54.853625+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: audit 2023-10-09T06:11:54.861397+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: cephadm 2023-10-09T06:11:54.894747+0000 mgr.y (mgr.25723) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: audit 2023-10-09T06:11:54.895232+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: audit 2023-10-09T06:11:54.896592+0000 mon.a (mon.0) 172 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:55 smithi125 bash[80002]: cephadm 2023-10-09T06:11:54.900545+0000 mgr.y (mgr.25723) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi125 2023-10-09T06:11:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: cluster 2023-10-09T06:11:54.518136+0000 mgr.y (mgr.25723) 150 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: audit 2023-10-09T06:11:54.853625+0000 mon.a (mon.0) 169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: audit 2023-10-09T06:11:54.861397+0000 mon.a (mon.0) 170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: cephadm 2023-10-09T06:11:54.894747+0000 mgr.y (mgr.25723) 151 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:11:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: audit 2023-10-09T06:11:54.895232+0000 mon.a (mon.0) 171 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:11:56.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: audit 2023-10-09T06:11:54.896592+0000 mon.a (mon.0) 172 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:56.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:55 smithi155 bash[59495]: cephadm 2023-10-09T06:11:54.900545+0000 mgr.y (mgr.25723) 152 : cephadm 1 Reconfiguring daemon osd.2 on smithi125 2023-10-09T06:11:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[77299]: cluster 2023-10-09T06:11:56.518893+0000 mgr.y (mgr.25723) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[77299]: audit 2023-10-09T06:11:57.060460+0000 mon.a (mon.0) 173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[77299]: audit 2023-10-09T06:11:57.068030+0000 mon.a (mon.0) 174 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[77299]: audit 2023-10-09T06:11:57.119654+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:11:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[77299]: audit 2023-10-09T06:11:57.121689+0000 mon.a (mon.0) 176 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[80002]: cluster 2023-10-09T06:11:56.518893+0000 mgr.y (mgr.25723) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[80002]: audit 2023-10-09T06:11:57.060460+0000 mon.a (mon.0) 173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[80002]: audit 2023-10-09T06:11:57.068030+0000 mon.a (mon.0) 174 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[80002]: audit 2023-10-09T06:11:57.119654+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:11:58.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:57 smithi125 bash[80002]: audit 2023-10-09T06:11:57.121689+0000 mon.a (mon.0) 176 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:57 smithi155 bash[59495]: cluster 2023-10-09T06:11:56.518893+0000 mgr.y (mgr.25723) 153 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:11:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:57 smithi155 bash[59495]: audit 2023-10-09T06:11:57.060460+0000 mon.a (mon.0) 173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:57 smithi155 bash[59495]: audit 2023-10-09T06:11:57.068030+0000 mon.a (mon.0) 174 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:11:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:57 smithi155 bash[59495]: audit 2023-10-09T06:11:57.119654+0000 mon.a (mon.0) 175 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:11:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:57 smithi155 bash[59495]: audit 2023-10-09T06:11:57.121689+0000 mon.a (mon.0) 176 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:59.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:58 smithi125 bash[77299]: cephadm 2023-10-09T06:11:57.119035+0000 mgr.y (mgr.25723) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:11:59.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:58 smithi125 bash[77299]: cephadm 2023-10-09T06:11:57.126257+0000 mgr.y (mgr.25723) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi125 2023-10-09T06:11:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:58 smithi125 bash[80002]: cephadm 2023-10-09T06:11:57.119035+0000 mgr.y (mgr.25723) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:11:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:58 smithi125 bash[80002]: cephadm 2023-10-09T06:11:57.126257+0000 mgr.y (mgr.25723) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi125 2023-10-09T06:11:59.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:58 smithi155 bash[59495]: cephadm 2023-10-09T06:11:57.119035+0000 mgr.y (mgr.25723) 154 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:11:59.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:58 smithi155 bash[59495]: cephadm 2023-10-09T06:11:57.126257+0000 mgr.y (mgr.25723) 155 : cephadm 1 Reconfiguring daemon osd.3 on smithi125 2023-10-09T06:12:00.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[77299]: cluster 2023-10-09T06:11:58.519491+0000 mgr.y (mgr.25723) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:00.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[77299]: audit 2023-10-09T06:11:59.398752+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[77299]: audit 2023-10-09T06:11:59.405848+0000 mon.a (mon.0) 178 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[77299]: audit 2023-10-09T06:11:59.447314+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:00.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[77299]: audit 2023-10-09T06:11:59.449288+0000 mon.a (mon.0) 180 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:00.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[80002]: cluster 2023-10-09T06:11:58.519491+0000 mgr.y (mgr.25723) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:00.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[80002]: audit 2023-10-09T06:11:59.398752+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[80002]: audit 2023-10-09T06:11:59.405848+0000 mon.a (mon.0) 178 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[80002]: audit 2023-10-09T06:11:59.447314+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:00.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:11:59 smithi125 bash[80002]: audit 2023-10-09T06:11:59.449288+0000 mon.a (mon.0) 180 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:59 smithi155 bash[59495]: cluster 2023-10-09T06:11:58.519491+0000 mgr.y (mgr.25723) 156 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:59 smithi155 bash[59495]: audit 2023-10-09T06:11:59.398752+0000 mon.a (mon.0) 177 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:59 smithi155 bash[59495]: audit 2023-10-09T06:11:59.405848+0000 mon.a (mon.0) 178 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:59 smithi155 bash[59495]: audit 2023-10-09T06:11:59.447314+0000 mon.a (mon.0) 179 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi125.yetyhy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:11:59 smithi155 bash[59495]: audit 2023-10-09T06:11:59.449288+0000 mon.a (mon.0) 180 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:01.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:00 smithi125 bash[80002]: cephadm 2023-10-09T06:11:59.446645+0000 mgr.y (mgr.25723) 157 : cephadm 1 Reconfiguring mds.foo.smithi125.yetyhy (monmap changed)... 2023-10-09T06:12:01.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:00 smithi125 bash[80002]: cephadm 2023-10-09T06:11:59.451254+0000 mgr.y (mgr.25723) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:12:01.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:00 smithi125 bash[77299]: cephadm 2023-10-09T06:11:59.446645+0000 mgr.y (mgr.25723) 157 : cephadm 1 Reconfiguring mds.foo.smithi125.yetyhy (monmap changed)... 2023-10-09T06:12:01.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:00 smithi125 bash[77299]: cephadm 2023-10-09T06:11:59.451254+0000 mgr.y (mgr.25723) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:12:01.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:00 smithi155 bash[59495]: cephadm 2023-10-09T06:11:59.446645+0000 mgr.y (mgr.25723) 157 : cephadm 1 Reconfiguring mds.foo.smithi125.yetyhy (monmap changed)... 2023-10-09T06:12:01.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:00 smithi155 bash[59495]: cephadm 2023-10-09T06:11:59.451254+0000 mgr.y (mgr.25723) 158 : cephadm 1 Reconfiguring daemon mds.foo.smithi125.yetyhy on smithi125 2023-10-09T06:12:01.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:01] "GET /metrics HTTP/1.1" 200 33915 "" "Prometheus/2.33.4" 2023-10-09T06:12:02.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:01 smithi125 bash[77299]: cluster 2023-10-09T06:12:00.520535+0000 mgr.y (mgr.25723) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:01 smithi125 bash[80002]: cluster 2023-10-09T06:12:00.520535+0000 mgr.y (mgr.25723) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:01 smithi155 bash[59495]: cluster 2023-10-09T06:12:00.520535+0000 mgr.y (mgr.25723) 159 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:03.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: audit 2023-10-09T06:12:01.905686+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: audit 2023-10-09T06:12:01.913447+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: cephadm 2023-10-09T06:12:01.920743+0000 mgr.y (mgr.25723) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T06:12:03.179 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: audit 2023-10-09T06:12:01.921213+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:12:03.179 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: audit 2023-10-09T06:12:01.922422+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:12:03.179 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: audit 2023-10-09T06:12:01.923655+0000 mon.a (mon.0) 185 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:03.179 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:02 smithi155 bash[59495]: cephadm 2023-10-09T06:12:01.925275+0000 mgr.y (mgr.25723) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi155 2023-10-09T06:12:03.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: audit 2023-10-09T06:12:01.905686+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: audit 2023-10-09T06:12:01.913447+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: cephadm 2023-10-09T06:12:01.920743+0000 mgr.y (mgr.25723) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T06:12:03.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: audit 2023-10-09T06:12:01.921213+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:12:03.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: audit 2023-10-09T06:12:01.922422+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: audit 2023-10-09T06:12:01.923655+0000 mon.a (mon.0) 185 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[77299]: cephadm 2023-10-09T06:12:01.925275+0000 mgr.y (mgr.25723) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi155 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: audit 2023-10-09T06:12:01.905686+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: audit 2023-10-09T06:12:01.913447+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: cephadm 2023-10-09T06:12:01.920743+0000 mgr.y (mgr.25723) 160 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: audit 2023-10-09T06:12:01.921213+0000 mon.a (mon.0) 183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: audit 2023-10-09T06:12:01.922422+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: audit 2023-10-09T06:12:01.923655+0000 mon.a (mon.0) 185 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:03.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:02 smithi125 bash[80002]: cephadm 2023-10-09T06:12:01.925275+0000 mgr.y (mgr.25723) 161 : cephadm 1 Reconfiguring daemon mon.b on smithi155 2023-10-09T06:12:04.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:03 smithi125 bash[77299]: cluster 2023-10-09T06:12:02.521059+0000 mgr.y (mgr.25723) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:04.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:03 smithi125 bash[77299]: audit 2023-10-09T06:12:03.552309+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:04.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:03 smithi125 bash[80002]: cluster 2023-10-09T06:12:02.521059+0000 mgr.y (mgr.25723) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:04.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:03 smithi125 bash[80002]: audit 2023-10-09T06:12:03.552309+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:03 smithi155 bash[59495]: cluster 2023-10-09T06:12:02.521059+0000 mgr.y (mgr.25723) 162 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:03 smithi155 bash[59495]: audit 2023-10-09T06:12:03.552309+0000 mon.a (mon.0) 186 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:05.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: audit 2023-10-09T06:12:04.114218+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: audit 2023-10-09T06:12:04.122676+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: cephadm 2023-10-09T06:12:04.144783+0000 mgr.y (mgr.25723) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: audit 2023-10-09T06:12:04.145575+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: audit 2023-10-09T06:12:04.147618+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: audit 2023-10-09T06:12:04.149389+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: cephadm 2023-10-09T06:12:04.151293+0000 mgr.y (mgr.25723) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi155 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[80002]: cluster 2023-10-09T06:12:04.522077+0000 mgr.y (mgr.25723) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: audit 2023-10-09T06:12:04.114218+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: audit 2023-10-09T06:12:04.122676+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: cephadm 2023-10-09T06:12:04.144783+0000 mgr.y (mgr.25723) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:12:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: audit 2023-10-09T06:12:04.145575+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:12:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: audit 2023-10-09T06:12:04.147618+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:12:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: audit 2023-10-09T06:12:04.149389+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: cephadm 2023-10-09T06:12:04.151293+0000 mgr.y (mgr.25723) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi155 2023-10-09T06:12:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:05 smithi125 bash[77299]: cluster 2023-10-09T06:12:04.522077+0000 mgr.y (mgr.25723) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: audit 2023-10-09T06:12:04.114218+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: audit 2023-10-09T06:12:04.122676+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: cephadm 2023-10-09T06:12:04.144783+0000 mgr.y (mgr.25723) 163 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:12:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: audit 2023-10-09T06:12:04.145575+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:12:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: audit 2023-10-09T06:12:04.147618+0000 mon.a (mon.0) 190 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:12:05.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: audit 2023-10-09T06:12:04.149389+0000 mon.a (mon.0) 191 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:05.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: cephadm 2023-10-09T06:12:04.151293+0000 mgr.y (mgr.25723) 164 : cephadm 1 Reconfiguring daemon mgr.x on smithi155 2023-10-09T06:12:05.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:05 smithi155 bash[59495]: cluster 2023-10-09T06:12:04.522077+0000 mgr.y (mgr.25723) 165 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:07.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: audit 2023-10-09T06:12:06.397888+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.785 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: audit 2023-10-09T06:12:06.405168+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: cephadm 2023-10-09T06:12:06.420930+0000 mgr.y (mgr.25723) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:12:07.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: audit 2023-10-09T06:12:06.421639+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:12:07.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: audit 2023-10-09T06:12:06.423301+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:07.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: cephadm 2023-10-09T06:12:06.427548+0000 mgr.y (mgr.25723) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi155 2023-10-09T06:12:07.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:07 smithi155 bash[59495]: cluster 2023-10-09T06:12:06.522709+0000 mgr.y (mgr.25723) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: audit 2023-10-09T06:12:06.397888+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: audit 2023-10-09T06:12:06.405168+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: cephadm 2023-10-09T06:12:06.420930+0000 mgr.y (mgr.25723) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:12:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: audit 2023-10-09T06:12:06.421639+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:12:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: audit 2023-10-09T06:12:06.423301+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: cephadm 2023-10-09T06:12:06.427548+0000 mgr.y (mgr.25723) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi155 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[77299]: cluster 2023-10-09T06:12:06.522709+0000 mgr.y (mgr.25723) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: audit 2023-10-09T06:12:06.397888+0000 mon.a (mon.0) 192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: audit 2023-10-09T06:12:06.405168+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: cephadm 2023-10-09T06:12:06.420930+0000 mgr.y (mgr.25723) 166 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: audit 2023-10-09T06:12:06.421639+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:12:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: audit 2023-10-09T06:12:06.423301+0000 mon.a (mon.0) 195 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:07.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: cephadm 2023-10-09T06:12:06.427548+0000 mgr.y (mgr.25723) 167 : cephadm 1 Reconfiguring daemon osd.4 on smithi155 2023-10-09T06:12:07.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:07 smithi125 bash[80002]: cluster 2023-10-09T06:12:06.522709+0000 mgr.y (mgr.25723) 168 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: cluster 2023-10-09T06:12:08.523352+0000 mgr.y (mgr.25723) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: audit 2023-10-09T06:12:08.589781+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: audit 2023-10-09T06:12:08.596359+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: cephadm 2023-10-09T06:12:08.646897+0000 mgr.y (mgr.25723) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: audit 2023-10-09T06:12:08.647505+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: audit 2023-10-09T06:12:08.649296+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[77299]: cephadm 2023-10-09T06:12:08.653668+0000 mgr.y (mgr.25723) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi155 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: cluster 2023-10-09T06:12:08.523352+0000 mgr.y (mgr.25723) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: audit 2023-10-09T06:12:08.589781+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: audit 2023-10-09T06:12:08.596359+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: cephadm 2023-10-09T06:12:08.646897+0000 mgr.y (mgr.25723) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:12:09.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: audit 2023-10-09T06:12:08.647505+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:12:09.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: audit 2023-10-09T06:12:08.649296+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:09.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:09 smithi125 bash[80002]: cephadm 2023-10-09T06:12:08.653668+0000 mgr.y (mgr.25723) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi155 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: cluster 2023-10-09T06:12:08.523352+0000 mgr.y (mgr.25723) 169 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: audit 2023-10-09T06:12:08.589781+0000 mon.a (mon.0) 196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: audit 2023-10-09T06:12:08.596359+0000 mon.a (mon.0) 197 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: cephadm 2023-10-09T06:12:08.646897+0000 mgr.y (mgr.25723) 170 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: audit 2023-10-09T06:12:08.647505+0000 mon.a (mon.0) 198 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:12:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: audit 2023-10-09T06:12:08.649296+0000 mon.a (mon.0) 199 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:09.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:09 smithi155 bash[59495]: cephadm 2023-10-09T06:12:08.653668+0000 mgr.y (mgr.25723) 171 : cephadm 1 Reconfiguring daemon osd.5 on smithi155 2023-10-09T06:12:11.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:11] "GET /metrics HTTP/1.1" 200 33919 "" "Prometheus/2.33.4" 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: cluster 2023-10-09T06:12:10.524377+0000 mgr.y (mgr.25723) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: audit 2023-10-09T06:12:10.845058+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: audit 2023-10-09T06:12:10.853585+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: cephadm 2023-10-09T06:12:10.872493+0000 mgr.y (mgr.25723) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: audit 2023-10-09T06:12:10.873082+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:12:12.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: audit 2023-10-09T06:12:10.874689+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[77299]: cephadm 2023-10-09T06:12:10.878844+0000 mgr.y (mgr.25723) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi155 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: cluster 2023-10-09T06:12:10.524377+0000 mgr.y (mgr.25723) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: audit 2023-10-09T06:12:10.845058+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: audit 2023-10-09T06:12:10.853585+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: cephadm 2023-10-09T06:12:10.872493+0000 mgr.y (mgr.25723) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: audit 2023-10-09T06:12:10.873082+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: audit 2023-10-09T06:12:10.874689+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:12.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:11 smithi125 bash[80002]: cephadm 2023-10-09T06:12:10.878844+0000 mgr.y (mgr.25723) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi155 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: cluster 2023-10-09T06:12:10.524377+0000 mgr.y (mgr.25723) 172 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: audit 2023-10-09T06:12:10.845058+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: audit 2023-10-09T06:12:10.853585+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: cephadm 2023-10-09T06:12:10.872493+0000 mgr.y (mgr.25723) 173 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: audit 2023-10-09T06:12:10.873082+0000 mon.a (mon.0) 202 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: audit 2023-10-09T06:12:10.874689+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:11 smithi155 bash[59495]: cephadm 2023-10-09T06:12:10.878844+0000 mgr.y (mgr.25723) 174 : cephadm 1 Reconfiguring daemon osd.6 on smithi155 2023-10-09T06:12:14.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[77299]: cluster 2023-10-09T06:12:12.525092+0000 mgr.y (mgr.25723) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[77299]: audit 2023-10-09T06:12:13.212061+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[77299]: audit 2023-10-09T06:12:13.219151+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[77299]: audit 2023-10-09T06:12:13.249162+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:12:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[77299]: audit 2023-10-09T06:12:13.251198+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[80002]: cluster 2023-10-09T06:12:12.525092+0000 mgr.y (mgr.25723) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[80002]: audit 2023-10-09T06:12:13.212061+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[80002]: audit 2023-10-09T06:12:13.219151+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[80002]: audit 2023-10-09T06:12:13.249162+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:12:14.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:13 smithi125 bash[80002]: audit 2023-10-09T06:12:13.251198+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:13 smithi155 bash[59495]: cluster 2023-10-09T06:12:12.525092+0000 mgr.y (mgr.25723) 175 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:13 smithi155 bash[59495]: audit 2023-10-09T06:12:13.212061+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:13 smithi155 bash[59495]: audit 2023-10-09T06:12:13.219151+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:13 smithi155 bash[59495]: audit 2023-10-09T06:12:13.249162+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:12:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:13 smithi155 bash[59495]: audit 2023-10-09T06:12:13.251198+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:15.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:14 smithi125 bash[77299]: cephadm 2023-10-09T06:12:13.248369+0000 mgr.y (mgr.25723) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:12:15.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:14 smithi125 bash[77299]: cephadm 2023-10-09T06:12:13.255870+0000 mgr.y (mgr.25723) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi155 2023-10-09T06:12:15.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:14 smithi125 bash[80002]: cephadm 2023-10-09T06:12:13.248369+0000 mgr.y (mgr.25723) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:12:15.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:14 smithi125 bash[80002]: cephadm 2023-10-09T06:12:13.255870+0000 mgr.y (mgr.25723) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi155 2023-10-09T06:12:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:14 smithi155 bash[59495]: cephadm 2023-10-09T06:12:13.248369+0000 mgr.y (mgr.25723) 176 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:12:15.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:14 smithi155 bash[59495]: cephadm 2023-10-09T06:12:13.255870+0000 mgr.y (mgr.25723) 177 : cephadm 1 Reconfiguring daemon osd.7 on smithi155 2023-10-09T06:12:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[77299]: cluster 2023-10-09T06:12:14.526114+0000 mgr.y (mgr.25723) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[77299]: audit 2023-10-09T06:12:15.390400+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[77299]: audit 2023-10-09T06:12:15.398206+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[77299]: audit 2023-10-09T06:12:15.426725+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[77299]: audit 2023-10-09T06:12:15.428660+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[80002]: cluster 2023-10-09T06:12:14.526114+0000 mgr.y (mgr.25723) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[80002]: audit 2023-10-09T06:12:15.390400+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[80002]: audit 2023-10-09T06:12:15.398206+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[80002]: audit 2023-10-09T06:12:15.426725+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:15 smithi125 bash[80002]: audit 2023-10-09T06:12:15.428660+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:15 smithi155 bash[59495]: cluster 2023-10-09T06:12:14.526114+0000 mgr.y (mgr.25723) 178 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:15 smithi155 bash[59495]: audit 2023-10-09T06:12:15.390400+0000 mon.a (mon.0) 208 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:15 smithi155 bash[59495]: audit 2023-10-09T06:12:15.398206+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:15 smithi155 bash[59495]: audit 2023-10-09T06:12:15.426725+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi155.itxbmb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:12:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:15 smithi155 bash[59495]: audit 2023-10-09T06:12:15.428660+0000 mon.a (mon.0) 211 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:17.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:16 smithi155 bash[59495]: cephadm 2023-10-09T06:12:15.425985+0000 mgr.y (mgr.25723) 179 : cephadm 1 Reconfiguring mds.foo.smithi155.itxbmb (monmap changed)... 2023-10-09T06:12:17.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:16 smithi155 bash[59495]: cephadm 2023-10-09T06:12:15.430589+0000 mgr.y (mgr.25723) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:12:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:16 smithi125 bash[77299]: cephadm 2023-10-09T06:12:15.425985+0000 mgr.y (mgr.25723) 179 : cephadm 1 Reconfiguring mds.foo.smithi155.itxbmb (monmap changed)... 2023-10-09T06:12:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:16 smithi125 bash[77299]: cephadm 2023-10-09T06:12:15.430589+0000 mgr.y (mgr.25723) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:12:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:16 smithi125 bash[80002]: cephadm 2023-10-09T06:12:15.425985+0000 mgr.y (mgr.25723) 179 : cephadm 1 Reconfiguring mds.foo.smithi155.itxbmb (monmap changed)... 2023-10-09T06:12:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:16 smithi125 bash[80002]: cephadm 2023-10-09T06:12:15.430589+0000 mgr.y (mgr.25723) 180 : cephadm 1 Reconfiguring daemon mds.foo.smithi155.itxbmb on smithi155 2023-10-09T06:12:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[77299]: cluster 2023-10-09T06:12:16.526839+0000 mgr.y (mgr.25723) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[77299]: audit 2023-10-09T06:12:17.703324+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[77299]: audit 2023-10-09T06:12:17.711833+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[77299]: audit 2023-10-09T06:12:17.762338+0000 mon.a (mon.0) 214 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[80002]: cluster 2023-10-09T06:12:16.526839+0000 mgr.y (mgr.25723) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[80002]: audit 2023-10-09T06:12:17.703324+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[80002]: audit 2023-10-09T06:12:17.711833+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:17 smithi125 bash[80002]: audit 2023-10-09T06:12:17.762338+0000 mon.a (mon.0) 214 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:17 smithi155 bash[59495]: cluster 2023-10-09T06:12:16.526839+0000 mgr.y (mgr.25723) 181 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:17 smithi155 bash[59495]: audit 2023-10-09T06:12:17.703324+0000 mon.a (mon.0) 212 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:17 smithi155 bash[59495]: audit 2023-10-09T06:12:17.711833+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:17 smithi155 bash[59495]: audit 2023-10-09T06:12:17.762338+0000 mon.a (mon.0) 214 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:19 smithi155 bash[59495]: audit 2023-10-09T06:12:18.139909+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:19 smithi155 bash[59495]: audit 2023-10-09T06:12:18.147517+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:19 smithi155 bash[59495]: cluster 2023-10-09T06:12:18.527435+0000 mgr.y (mgr.25723) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:19 smithi155 bash[59495]: audit 2023-10-09T06:12:18.552393+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[77299]: audit 2023-10-09T06:12:18.139909+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[77299]: audit 2023-10-09T06:12:18.147517+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[77299]: cluster 2023-10-09T06:12:18.527435+0000 mgr.y (mgr.25723) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[77299]: audit 2023-10-09T06:12:18.552393+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[80002]: audit 2023-10-09T06:12:18.139909+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[80002]: audit 2023-10-09T06:12:18.147517+0000 mon.a (mon.0) 216 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[80002]: cluster 2023-10-09T06:12:18.527435+0000 mgr.y (mgr.25723) 182 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:19.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:19 smithi125 bash[80002]: audit 2023-10-09T06:12:18.552393+0000 mon.a (mon.0) 217 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:21.587 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:21] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.33.4" 2023-10-09T06:12:21.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:21 smithi125 bash[80002]: cluster 2023-10-09T06:12:20.528501+0000 mgr.y (mgr.25723) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:21 smithi125 bash[77299]: cluster 2023-10-09T06:12:20.528501+0000 mgr.y (mgr.25723) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:21 smithi155 bash[59495]: cluster 2023-10-09T06:12:20.528501+0000 mgr.y (mgr.25723) 183 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:23.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:23 smithi125 bash[77299]: cluster 2023-10-09T06:12:22.529240+0000 mgr.y (mgr.25723) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:23.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:23 smithi125 bash[80002]: cluster 2023-10-09T06:12:22.529240+0000 mgr.y (mgr.25723) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:23.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:23 smithi155 bash[59495]: cluster 2023-10-09T06:12:22.529240+0000 mgr.y (mgr.25723) 184 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: audit 2023-10-09T06:12:24.189188+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: audit 2023-10-09T06:12:24.198275+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: cluster 2023-10-09T06:12:24.529870+0000 mgr.y (mgr.25723) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: audit 2023-10-09T06:12:24.679169+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: audit 2023-10-09T06:12:24.680931+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[77299]: audit 2023-10-09T06:12:24.691273+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: audit 2023-10-09T06:12:24.189188+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: audit 2023-10-09T06:12:24.198275+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: cluster 2023-10-09T06:12:24.529870+0000 mgr.y (mgr.25723) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: audit 2023-10-09T06:12:24.679169+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: audit 2023-10-09T06:12:24.680931+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:25.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:25 smithi125 bash[80002]: audit 2023-10-09T06:12:24.691273+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: audit 2023-10-09T06:12:24.189188+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: audit 2023-10-09T06:12:24.198275+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: cluster 2023-10-09T06:12:24.529870+0000 mgr.y (mgr.25723) 185 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: audit 2023-10-09T06:12:24.679169+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: audit 2023-10-09T06:12:24.680931+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:25 smithi155 bash[59495]: audit 2023-10-09T06:12:24.691273+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:27.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:27 smithi125 bash[77299]: cluster 2023-10-09T06:12:26.530445+0000 mgr.y (mgr.25723) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:27 smithi125 bash[80002]: cluster 2023-10-09T06:12:26.530445+0000 mgr.y (mgr.25723) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:27 smithi155 bash[59495]: cluster 2023-10-09T06:12:26.530445+0000 mgr.y (mgr.25723) 186 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:29 smithi125 bash[77299]: cluster 2023-10-09T06:12:28.531040+0000 mgr.y (mgr.25723) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:29.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:29 smithi125 bash[80002]: cluster 2023-10-09T06:12:28.531040+0000 mgr.y (mgr.25723) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:29 smithi155 bash[59495]: cluster 2023-10-09T06:12:28.531040+0000 mgr.y (mgr.25723) 187 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:31.591 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:31] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.33.4" 2023-10-09T06:12:31.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:31 smithi125 bash[77299]: cluster 2023-10-09T06:12:30.532054+0000 mgr.y (mgr.25723) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:31 smithi125 bash[80002]: cluster 2023-10-09T06:12:30.532054+0000 mgr.y (mgr.25723) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:31 smithi155 bash[59495]: cluster 2023-10-09T06:12:30.532054+0000 mgr.y (mgr.25723) 188 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:33 smithi125 bash[77299]: cluster 2023-10-09T06:12:32.532664+0000 mgr.y (mgr.25723) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:33 smithi125 bash[77299]: audit 2023-10-09T06:12:33.553049+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:33 smithi125 bash[80002]: cluster 2023-10-09T06:12:32.532664+0000 mgr.y (mgr.25723) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:33 smithi125 bash[80002]: audit 2023-10-09T06:12:33.553049+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:33 smithi155 bash[59495]: cluster 2023-10-09T06:12:32.532664+0000 mgr.y (mgr.25723) 189 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:33 smithi155 bash[59495]: audit 2023-10-09T06:12:33.553049+0000 mon.a (mon.0) 223 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:35.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:35 smithi125 bash[80002]: cluster 2023-10-09T06:12:34.533757+0000 mgr.y (mgr.25723) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:35.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:35 smithi125 bash[77299]: cluster 2023-10-09T06:12:34.533757+0000 mgr.y (mgr.25723) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:35 smithi155 bash[59495]: cluster 2023-10-09T06:12:34.533757+0000 mgr.y (mgr.25723) 190 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:37 smithi125 bash[77299]: cluster 2023-10-09T06:12:36.534459+0000 mgr.y (mgr.25723) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:37 smithi125 bash[80002]: cluster 2023-10-09T06:12:36.534459+0000 mgr.y (mgr.25723) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:37 smithi155 bash[59495]: cluster 2023-10-09T06:12:36.534459+0000 mgr.y (mgr.25723) 191 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:39 smithi125 bash[77299]: cluster 2023-10-09T06:12:38.535187+0000 mgr.y (mgr.25723) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:39 smithi125 bash[80002]: cluster 2023-10-09T06:12:38.535187+0000 mgr.y (mgr.25723) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:39 smithi155 bash[59495]: cluster 2023-10-09T06:12:38.535187+0000 mgr.y (mgr.25723) 192 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:41.576 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:41] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-10-09T06:12:41.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:41 smithi125 bash[77299]: cluster 2023-10-09T06:12:40.536279+0000 mgr.y (mgr.25723) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:41.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:41 smithi125 bash[80002]: cluster 2023-10-09T06:12:40.536279+0000 mgr.y (mgr.25723) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:41.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:41 smithi155 bash[59495]: cluster 2023-10-09T06:12:40.536279+0000 mgr.y (mgr.25723) 193 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:42.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:12:43.762 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:43 smithi125 bash[77299]: cluster 2023-10-09T06:12:42.536945+0000 mgr.y (mgr.25723) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:43.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:43 smithi125 bash[80002]: cluster 2023-10-09T06:12:42.536945+0000 mgr.y (mgr.25723) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:43 smithi155 bash[59495]: cluster 2023-10-09T06:12:42.536945+0000 mgr.y (mgr.25723) 194 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:45.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:45 smithi125 bash[80002]: cluster 2023-10-09T06:12:44.538023+0000 mgr.y (mgr.25723) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:45.743 INFO:teuthology.orchestra.run.smithi125.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:12:45.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:45 smithi125 bash[77299]: cluster 2023-10-09T06:12:44.538023+0000 mgr.y (mgr.25723) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:45 smithi155 bash[59495]: cluster 2023-10-09T06:12:44.538023+0000 mgr.y (mgr.25723) 195 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:46.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T06:12:46.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[77299]: audit 2023-10-09T06:12:45.734166+0000 mgr.y (mgr.25723) 196 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[77299]: cephadm 2023-10-09T06:12:45.734800+0000 mgr.y (mgr.25723) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[77299]: audit 2023-10-09T06:12:45.739476+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[77299]: audit 2023-10-09T06:12:45.743386+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[80002]: audit 2023-10-09T06:12:45.734166+0000 mgr.y (mgr.25723) 196 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[80002]: cephadm 2023-10-09T06:12:45.734800+0000 mgr.y (mgr.25723) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[80002]: audit 2023-10-09T06:12:45.739476+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:46.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:46 smithi125 bash[80002]: audit 2023-10-09T06:12:45.743386+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:46 smithi155 bash[59495]: audit 2023-10-09T06:12:45.734166+0000 mgr.y (mgr.25723) 196 : audit 0 from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:12:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:46 smithi155 bash[59495]: cephadm 2023-10-09T06:12:45.734800+0000 mgr.y (mgr.25723) 197 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:12:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:46 smithi155 bash[59495]: audit 2023-10-09T06:12:45.739476+0000 mon.a (mon.0) 224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:46 smithi155 bash[59495]: audit 2023-10-09T06:12:45.743386+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:48.061 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:47 smithi125 bash[77299]: cluster 2023-10-09T06:12:46.538726+0000 mgr.y (mgr.25723) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:48.062 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:47 smithi125 bash[80002]: cluster 2023-10-09T06:12:46.538726+0000 mgr.y (mgr.25723) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:48.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:47 smithi155 bash[59495]: cluster 2023-10-09T06:12:46.538726+0000 mgr.y (mgr.25723) 198 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:49.642 INFO:teuthology.orchestra.run.smithi125.stdout:wait for half of osds to upgrade 2023-10-09T06:12:49.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:49 smithi125 bash[77299]: cluster 2023-10-09T06:12:48.539331+0000 mgr.y (mgr.25723) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:49.744 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:49 smithi125 bash[77299]: audit 2023-10-09T06:12:48.553220+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:49.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:49 smithi125 bash[80002]: cluster 2023-10-09T06:12:48.539331+0000 mgr.y (mgr.25723) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:49.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:49 smithi125 bash[80002]: audit 2023-10-09T06:12:48.553220+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:49 smithi155 bash[59495]: cluster 2023-10-09T06:12:48.539331+0000 mgr.y (mgr.25723) 199 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:49 smithi155 bash[59495]: audit 2023-10-09T06:12:48.553220+0000 mon.a (mon.0) 226 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:50.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T06:12:51.601 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:12:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:12:51] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-10-09T06:12:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:51 smithi125 bash[80002]: cluster 2023-10-09T06:12:50.540033+0000 mgr.y (mgr.25723) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:51 smithi125 bash[77299]: cluster 2023-10-09T06:12:50.540033+0000 mgr.y (mgr.25723) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:51 smithi155 bash[59495]: cluster 2023-10-09T06:12:50.540033+0000 mgr.y (mgr.25723) 200 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:53.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:51.936839+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:51.947248+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.417592+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.418752+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.426735+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.513436+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.517966+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.520381+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.528967+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.530532+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:12:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.539913+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.542615+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.549290+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.550411+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.557870+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.560105+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.569104+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[77299]: audit 2023-10-09T06:12:52.571749+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:51.936839+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:51.947248+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.417592+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.418752+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.426735+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.513436+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.517966+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.520381+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.528967+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.530532+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.539913+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.542615+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:12:53.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.549290+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:12:53.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.550411+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:12:53.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.557870+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:12:53.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.560105+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.569104+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:52 smithi125 bash[80002]: audit 2023-10-09T06:12:52.571749+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:51.936839+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:51.947248+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.417592+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.418752+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.426735+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.513436+0000 mon.a (mon.0) 232 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.517966+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.520381+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.528967+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.530532+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:12:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.539913+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.542615+0000 mon.a (mon.0) 238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.549290+0000 mon.a (mon.0) 239 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.550411+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.557870+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.560105+0000 mon.a (mon.0) 242 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.569104+0000 mon.a (mon.0) 243 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:53.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:52 smithi155 bash[59495]: audit 2023-10-09T06:12:52.571749+0000 mon.a (mon.0) 244 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[77299]: cephadm 2023-10-09T06:12:52.521422+0000 mgr.y (mgr.25723) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T06:12:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[77299]: cluster 2023-10-09T06:12:52.540538+0000 mgr.y (mgr.25723) 202 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[77299]: cephadm 2023-10-09T06:12:52.561528+0000 mgr.y (mgr.25723) 203 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T06:12:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[77299]: audit 2023-10-09T06:12:52.571981+0000 mgr.y (mgr.25723) 204 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[77299]: cephadm 2023-10-09T06:12:52.573282+0000 mgr.y (mgr.25723) 205 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T06:12:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[80002]: cephadm 2023-10-09T06:12:52.521422+0000 mgr.y (mgr.25723) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T06:12:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[80002]: cluster 2023-10-09T06:12:52.540538+0000 mgr.y (mgr.25723) 202 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[80002]: cephadm 2023-10-09T06:12:52.561528+0000 mgr.y (mgr.25723) 203 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T06:12:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[80002]: audit 2023-10-09T06:12:52.571981+0000 mgr.y (mgr.25723) 204 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:53 smithi125 bash[80002]: cephadm 2023-10-09T06:12:52.573282+0000 mgr.y (mgr.25723) 205 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T06:12:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:53 smithi155 bash[59495]: cephadm 2023-10-09T06:12:52.521422+0000 mgr.y (mgr.25723) 201 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-09T06:12:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:53 smithi155 bash[59495]: cluster 2023-10-09T06:12:52.540538+0000 mgr.y (mgr.25723) 202 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:53 smithi155 bash[59495]: cephadm 2023-10-09T06:12:52.561528+0000 mgr.y (mgr.25723) 203 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-09T06:12:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:53 smithi155 bash[59495]: audit 2023-10-09T06:12:52.571981+0000 mgr.y (mgr.25723) 204 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:12:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:53 smithi155 bash[59495]: cephadm 2023-10-09T06:12:52.573282+0000 mgr.y (mgr.25723) 205 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-09T06:12:55.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:54 smithi125 bash[77299]: audit 2023-10-09T06:12:54.538990+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.125:0/247296145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:55.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:54 smithi125 bash[80002]: audit 2023-10-09T06:12:54.538990+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.125:0/247296145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:54 smithi155 bash[59495]: audit 2023-10-09T06:12:54.538990+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.125:0/247296145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:56.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[77299]: cluster 2023-10-09T06:12:54.541522+0000 mgr.y (mgr.25723) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:56.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[77299]: audit 2023-10-09T06:12:55.204368+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:56.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[77299]: audit 2023-10-09T06:12:55.205923+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:12:56.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[77299]: audit 2023-10-09T06:12:55.207047+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:56.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[80002]: cluster 2023-10-09T06:12:54.541522+0000 mgr.y (mgr.25723) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:56.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[80002]: audit 2023-10-09T06:12:55.204368+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:56.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[80002]: audit 2023-10-09T06:12:55.205923+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:12:56.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:55 smithi125 bash[80002]: audit 2023-10-09T06:12:55.207047+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:55 smithi155 bash[59495]: cluster 2023-10-09T06:12:54.541522+0000 mgr.y (mgr.25723) 206 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:55 smithi155 bash[59495]: audit 2023-10-09T06:12:55.204368+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:12:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:55 smithi155 bash[59495]: audit 2023-10-09T06:12:55.205923+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:12:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:55 smithi155 bash[59495]: audit 2023-10-09T06:12:55.207047+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[77299]: cephadm 2023-10-09T06:12:55.193388+0000 mgr.y (mgr.25723) 207 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T06:12:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[77299]: cephadm 2023-10-09T06:12:55.210893+0000 mgr.y (mgr.25723) 208 : cephadm 1 Deploying daemon osd.0 on smithi125 2023-10-09T06:12:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[77299]: audit 2023-10-09T06:12:56.902092+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.125:0/2305860156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:57.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[80002]: cephadm 2023-10-09T06:12:55.193388+0000 mgr.y (mgr.25723) 207 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T06:12:57.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[80002]: cephadm 2023-10-09T06:12:55.210893+0000 mgr.y (mgr.25723) 208 : cephadm 1 Deploying daemon osd.0 on smithi125 2023-10-09T06:12:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:56 smithi125 bash[80002]: audit 2023-10-09T06:12:56.902092+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.125:0/2305860156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:56 smithi155 bash[59495]: cephadm 2023-10-09T06:12:55.193388+0000 mgr.y (mgr.25723) 207 : cephadm 1 Upgrade: Updating osd.0 2023-10-09T06:12:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:56 smithi155 bash[59495]: cephadm 2023-10-09T06:12:55.210893+0000 mgr.y (mgr.25723) 208 : cephadm 1 Deploying daemon osd.0 on smithi125 2023-10-09T06:12:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:56 smithi155 bash[59495]: audit 2023-10-09T06:12:56.902092+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.125:0/2305860156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:58.235 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:57 smithi125 bash[77299]: cluster 2023-10-09T06:12:56.542148+0000 mgr.y (mgr.25723) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:58.235 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:57 smithi125 bash[80002]: cluster 2023-10-09T06:12:56.542148+0000 mgr.y (mgr.25723) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:57 smithi155 bash[59495]: cluster 2023-10-09T06:12:56.542148+0000 mgr.y (mgr.25723) 209 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:59.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:59 smithi125 bash[77299]: cluster 2023-10-09T06:12:58.542781+0000 mgr.y (mgr.25723) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:59.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:12:59 smithi125 bash[77299]: audit 2023-10-09T06:12:59.251550+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.125:0/1022324586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:59.866 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:59 smithi125 bash[80002]: cluster 2023-10-09T06:12:58.542781+0000 mgr.y (mgr.25723) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:59.866 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:12:59 smithi125 bash[80002]: audit 2023-10-09T06:12:59.251550+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.125:0/1022324586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:59 smithi155 bash[59495]: cluster 2023-10-09T06:12:58.542781+0000 mgr.y (mgr.25723) 210 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:12:59 smithi155 bash[59495]: audit 2023-10-09T06:12:59.251550+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.125:0/1022324586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:00.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:12:59 smithi125 systemd[1]: Stopping Ceph osd.0 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:13:00.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:00 smithi125 bash[37018]: debug 2023-10-09T06:13:00.007+0000 7f862af2a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:13:00.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:00 smithi125 bash[37018]: debug 2023-10-09T06:13:00.007+0000 7f862af2a700 -1 osd.0 84 *** Got signal Terminated *** 2023-10-09T06:13:00.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:00 smithi125 bash[37018]: debug 2023-10-09T06:13:00.007+0000 7f862af2a700 -1 osd.0 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:13:00.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.014090+0000 mon.a (mon.0) 248 : cluster 1 osd.0 marked itself down and dead 2023-10-09T06:13:00.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.014090+0000 mon.a (mon.0) 248 : cluster 1 osd.0 marked itself down and dead 2023-10-09T06:13:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.014090+0000 mon.a (mon.0) 248 : cluster 1 osd.0 marked itself down and dead 2023-10-09T06:13:01.475 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[85234]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T06:13:01.476 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[85354]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T06:13:01.476 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:01] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-10-09T06:13:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.491866+0000 mon.a (mon.0) 249 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.504793+0000 mon.a (mon.0) 250 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T06:13:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.505042+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.506615+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[77299]: cluster 2023-10-09T06:13:00.543369+0000 mgr.y (mgr.25723) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.491866+0000 mon.a (mon.0) 249 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.504793+0000 mon.a (mon.0) 250 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T06:13:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.505042+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.506615+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:01 smithi125 bash[80002]: cluster 2023-10-09T06:13:00.543369+0000 mgr.y (mgr.25723) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:01 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.491866+0000 mon.a (mon.0) 249 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:01 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.504793+0000 mon.a (mon.0) 250 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-09T06:13:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:01 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.505042+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:01 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.506615+0000 osd.2 (osd.2) 4 : cluster 3 failed to encode map e85 with expected crc 2023-10-09T06:13:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:01 smithi155 bash[59495]: cluster 2023-10-09T06:13:00.543369+0000 mgr.y (mgr.25723) 211 : cluster 0 pgmap v145: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.511258+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.511277+0000 mon.a (mon.0) 251 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T06:13:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.511316+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.511682+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.511805+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.513095+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.513788+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.514157+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.514270+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: cluster 2023-10-09T06:13:01.515225+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[77299]: audit 2023-10-09T06:13:01.616711+0000 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.125:0/4108080013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.511258+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.511277+0000 mon.a (mon.0) 251 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.511316+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.511682+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.511805+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.513095+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.513788+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.514157+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.514270+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: cluster 2023-10-09T06:13:01.515225+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:02 smithi125 bash[80002]: audit 2023-10-09T06:13:01.616711+0000 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.125:0/4108080013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.511258+0000 osd.6 (osd.6) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.511277+0000 mon.a (mon.0) 251 : cluster 0 osdmap e86: 8 total, 7 up, 8 in 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.511316+0000 osd.4 (osd.4) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.511682+0000 osd.1 (osd.1) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.511805+0000 osd.3 (osd.3) 3 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.513095+0000 osd.2 (osd.2) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.513788+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.514157+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.514270+0000 osd.5 (osd.5) 4 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: cluster 2023-10-09T06:13:01.515225+0000 osd.4 (osd.4) 6 : cluster 3 failed to encode map e86 with expected crc 2023-10-09T06:13:02.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:02 smithi155 bash[59495]: audit 2023-10-09T06:13:01.616711+0000 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.125:0/4108080013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:03.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:03 smithi125 bash[77299]: cluster 2023-10-09T06:13:02.543837+0000 mgr.y (mgr.25723) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:03.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:03 smithi125 bash[80002]: cluster 2023-10-09T06:13:02.543837+0000 mgr.y (mgr.25723) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:03 smithi155 bash[59495]: cluster 2023-10-09T06:13:02.543837+0000 mgr.y (mgr.25723) 212 : cluster 0 pgmap v147: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:04.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:03 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T06:13:04.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:03 smithi125 systemd[1]: Stopped Ceph osd.0 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:13:04.137 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:04 smithi125 systemd[1]: Started Ceph osd.0 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:03.560228+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:03.561300+0000 mon.a (mon.0) 254 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:03.978436+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.125:0/190408366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:04.089547+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:04.096945+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[80002]: audit 2023-10-09T06:13:04.143463+0000 mon.a (mon.0) 257 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:03.560228+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:03.561300+0000 mon.a (mon.0) 254 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:03.978436+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.125:0/190408366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:04.089547+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:04.096945+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:04 smithi125 bash[77299]: audit 2023-10-09T06:13:04.143463+0000 mon.a (mon.0) 257 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:03.560228+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:03.561300+0000 mon.a (mon.0) 254 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:03.978436+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.125:0/190408366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:04.089547+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:04.096945+0000 mon.a (mon.0) 256 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:04 smithi155 bash[59495]: audit 2023-10-09T06:13:04.143463+0000 mon.a (mon.0) 257 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:05.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[77299]: cluster 2023-10-09T06:13:04.545072+0000 mgr.y (mgr.25723) 213 : cluster 0 pgmap v148: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:05.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[77299]: cluster 2023-10-09T06:13:05.093990+0000 mon.a (mon.0) 258 : cluster 3 Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:05.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[80002]: cluster 2023-10-09T06:13:04.545072+0000 mgr.y (mgr.25723) 213 : cluster 0 pgmap v148: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:05.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[80002]: cluster 2023-10-09T06:13:05.093990+0000 mon.a (mon.0) 258 : cluster 3 Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:05.859 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:13:05.859 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:13:05.860 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:05 smithi125 bash[85540]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T06:13:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:05 smithi155 bash[59495]: cluster 2023-10-09T06:13:04.545072+0000 mgr.y (mgr.25723) 213 : cluster 0 pgmap v148: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:05 smithi155 bash[59495]: cluster 2023-10-09T06:13:05.093990+0000 mon.a (mon.0) 258 : cluster 3 Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:06.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:06 smithi125 bash[80002]: audit 2023-10-09T06:13:06.329671+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.125:0/2725961725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:06.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:06 smithi125 bash[77299]: audit 2023-10-09T06:13:06.329671+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.125:0/2725961725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:06 smithi155 bash[59495]: audit 2023-10-09T06:13:06.329671+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.125:0/2725961725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:07 smithi125 bash[77299]: cluster 2023-10-09T06:13:06.545826+0000 mgr.y (mgr.25723) 214 : cluster 0 pgmap v149: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:07 smithi125 bash[80002]: cluster 2023-10-09T06:13:06.545826+0000 mgr.y (mgr.25723) 214 : cluster 0 pgmap v149: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:07.888 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:07 smithi125 bash[85861]: debug 2023-10-09T06:13:07.595+0000 7fc8f9708700 1 -- 172.21.15.125:0/2490834886 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d8b32da4e0 con 0x55d8b3334c00 2023-10-09T06:13:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:07 smithi155 bash[59495]: cluster 2023-10-09T06:13:06.545826+0000 mgr.y (mgr.25723) 214 : cluster 0 pgmap v149: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:08.637 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:08 smithi125 bash[85861]: debug 2023-10-09T06:13:08.139+0000 7fc90288e640 -1 Falling back to public interface 2023-10-09T06:13:09.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:09 smithi125 bash[80002]: cluster 2023-10-09T06:13:08.546548+0000 mgr.y (mgr.25723) 215 : cluster 0 pgmap v150: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:09.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:09 smithi125 bash[80002]: audit 2023-10-09T06:13:08.694134+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.125:0/1102732991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:09 smithi125 bash[77299]: cluster 2023-10-09T06:13:08.546548+0000 mgr.y (mgr.25723) 215 : cluster 0 pgmap v150: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:09 smithi125 bash[77299]: audit 2023-10-09T06:13:08.694134+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.125:0/1102732991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:09 smithi155 bash[59495]: cluster 2023-10-09T06:13:08.546548+0000 mgr.y (mgr.25723) 215 : cluster 0 pgmap v150: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:09 smithi155 bash[59495]: audit 2023-10-09T06:13:08.694134+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.125:0/1102732991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:11.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:11] "GET /metrics HTTP/1.1" 200 33983 "" "Prometheus/2.33.4" 2023-10-09T06:13:11.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[77299]: cluster 2023-10-09T06:13:10.547708+0000 mgr.y (mgr.25723) 216 : cluster 0 pgmap v151: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:11.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[77299]: audit 2023-10-09T06:13:11.078096+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.125:0/1393332495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:11.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[77299]: audit 2023-10-09T06:13:11.190251+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:11.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[77299]: audit 2023-10-09T06:13:11.198840+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:11.888 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[85861]: debug 2023-10-09T06:13:11.635+0000 7fc90288e640 -1 osd.0 84 log_to_monitors true 2023-10-09T06:13:11.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[80002]: cluster 2023-10-09T06:13:10.547708+0000 mgr.y (mgr.25723) 216 : cluster 0 pgmap v151: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:11.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[80002]: audit 2023-10-09T06:13:11.078096+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.125:0/1393332495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:11.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[80002]: audit 2023-10-09T06:13:11.190251+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:11.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:11 smithi125 bash[80002]: audit 2023-10-09T06:13:11.198840+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:11 smithi155 bash[59495]: cluster 2023-10-09T06:13:10.547708+0000 mgr.y (mgr.25723) 216 : cluster 0 pgmap v151: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:11 smithi155 bash[59495]: audit 2023-10-09T06:13:11.078096+0000 mon.c (mon.2) 12 : audit 0 from='client.? 172.21.15.125:0/1393332495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:11 smithi155 bash[59495]: audit 2023-10-09T06:13:11.190251+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:11 smithi155 bash[59495]: audit 2023-10-09T06:13:11.198840+0000 mon.a (mon.0) 260 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.887 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[85861]: debug 2023-10-09T06:13:12.647+0000 7fc8f14f5700 -1 osd.0 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:13:12.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[80002]: audit 2023-10-09T06:13:11.645700+0000 mon.a (mon.0) 261 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:13:12.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[80002]: audit 2023-10-09T06:13:11.991924+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[80002]: audit 2023-10-09T06:13:11.999119+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[77299]: audit 2023-10-09T06:13:11.645700+0000 mon.a (mon.0) 261 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:13:12.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[77299]: audit 2023-10-09T06:13:11.991924+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:12 smithi125 bash[77299]: audit 2023-10-09T06:13:11.999119+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:12 smithi155 bash[59495]: audit 2023-10-09T06:13:11.645700+0000 mon.a (mon.0) 261 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:13:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:12 smithi155 bash[59495]: audit 2023-10-09T06:13:11.991924+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:12.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:12 smithi155 bash[59495]: audit 2023-10-09T06:13:11.999119+0000 mon.a (mon.0) 263 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: cluster 2023-10-09T06:13:12.548222+0000 mgr.y (mgr.25723) 217 : cluster 0 pgmap v152: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: cluster 2023-10-09T06:13:12.612838+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: audit 2023-10-09T06:13:12.614811+0000 mon.a (mon.0) 264 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: cluster 2023-10-09T06:13:12.614880+0000 mon.a (mon.0) 265 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: cluster 2023-10-09T06:13:12.615474+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: audit 2023-10-09T06:13:12.615524+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[80002]: audit 2023-10-09T06:13:13.443845+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.125:0/4002314491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: cluster 2023-10-09T06:13:12.548222+0000 mgr.y (mgr.25723) 217 : cluster 0 pgmap v152: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: cluster 2023-10-09T06:13:12.612838+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: audit 2023-10-09T06:13:12.614811+0000 mon.a (mon.0) 264 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: cluster 2023-10-09T06:13:12.614880+0000 mon.a (mon.0) 265 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: cluster 2023-10-09T06:13:12.615474+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: audit 2023-10-09T06:13:12.615524+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:13.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:13 smithi125 bash[77299]: audit 2023-10-09T06:13:13.443845+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.125:0/4002314491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: cluster 2023-10-09T06:13:12.548222+0000 mgr.y (mgr.25723) 217 : cluster 0 pgmap v152: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: cluster 2023-10-09T06:13:12.612838+0000 osd.1 (osd.1) 5 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: audit 2023-10-09T06:13:12.614811+0000 mon.a (mon.0) 264 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: cluster 2023-10-09T06:13:12.614880+0000 mon.a (mon.0) 265 : cluster 0 osdmap e87: 8 total, 7 up, 8 in 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: cluster 2023-10-09T06:13:12.615474+0000 osd.4 (osd.4) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-09T06:13:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: audit 2023-10-09T06:13:12.615524+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:13.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:13 smithi155 bash[59495]: audit 2023-10-09T06:13:13.443845+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.125:0/4002314491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: cluster 2023-10-09T06:13:13.612570+0000 mon.a (mon.0) 267 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: cluster 2023-10-09T06:13:13.616041+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:14.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: cluster 2023-10-09T06:13:13.617444+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:14.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: cluster 2023-10-09T06:13:13.620038+0000 mon.a (mon.0) 268 : cluster 1 osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221] boot 2023-10-09T06:13:14.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: cluster 2023-10-09T06:13:13.620130+0000 mon.a (mon.0) 269 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T06:13:14.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:14 smithi155 bash[59495]: audit 2023-10-09T06:13:13.620879+0000 mon.a (mon.0) 270 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:13:15.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: cluster 2023-10-09T06:13:13.612570+0000 mon.a (mon.0) 267 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: cluster 2023-10-09T06:13:13.616041+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: cluster 2023-10-09T06:13:13.617444+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: cluster 2023-10-09T06:13:13.620038+0000 mon.a (mon.0) 268 : cluster 1 osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221] boot 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: cluster 2023-10-09T06:13:13.620130+0000 mon.a (mon.0) 269 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[77299]: audit 2023-10-09T06:13:13.620879+0000 mon.a (mon.0) 270 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:13:15.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: cluster 2023-10-09T06:13:13.612570+0000 mon.a (mon.0) 267 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:15.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: cluster 2023-10-09T06:13:13.616041+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:15.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: cluster 2023-10-09T06:13:13.617444+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e88 with expected crc 2023-10-09T06:13:15.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: cluster 2023-10-09T06:13:13.620038+0000 mon.a (mon.0) 268 : cluster 1 osd.0 [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221] boot 2023-10-09T06:13:15.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: cluster 2023-10-09T06:13:13.620130+0000 mon.a (mon.0) 269 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-09T06:13:15.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:14 smithi125 bash[80002]: audit 2023-10-09T06:13:13.620879+0000 mon.a (mon.0) 270 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:13:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.549174+0000 mgr.y (mgr.25723) 218 : cluster 0 pgmap v155: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.623213+0000 mon.a (mon.0) 271 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.627976+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.631582+0000 mon.a (mon.0) 272 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.631966+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.632327+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.632844+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.633487+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.634170+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.634400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.635773+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.637179+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.638888+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:15.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:15 smithi155 bash[59495]: cluster 2023-10-09T06:13:14.640688+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.549174+0000 mgr.y (mgr.25723) 218 : cluster 0 pgmap v155: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.623213+0000 mon.a (mon.0) 271 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.627976+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.631582+0000 mon.a (mon.0) 272 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.631966+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.632327+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.632844+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.633487+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.634170+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.634400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.635773+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.637179+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.638888+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[77299]: cluster 2023-10-09T06:13:14.640688+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.549174+0000 mgr.y (mgr.25723) 218 : cluster 0 pgmap v155: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.623213+0000 mon.a (mon.0) 271 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.627976+0000 osd.1 (osd.1) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.631582+0000 mon.a (mon.0) 272 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.631966+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.632327+0000 osd.5 (osd.5) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.632844+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.633487+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.634170+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.634400+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.635773+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.637179+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.638888+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:16.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:15 smithi125 bash[80002]: cluster 2023-10-09T06:13:14.640688+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e89 with expected crc 2023-10-09T06:13:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:16 smithi125 bash[77299]: audit 2023-10-09T06:13:15.806908+0000 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.125:0/3863667362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:16 smithi125 bash[80002]: audit 2023-10-09T06:13:15.806908+0000 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.125:0/3863667362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:16 smithi155 bash[59495]: audit 2023-10-09T06:13:15.806908+0000 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.125:0/3863667362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.922 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:17 smithi125 bash[77299]: cluster 2023-10-09T06:13:16.549880+0000 mgr.y (mgr.25723) 219 : cluster 0 pgmap v157: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:17.923 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:17 smithi125 bash[80002]: cluster 2023-10-09T06:13:16.549880+0000 mgr.y (mgr.25723) 219 : cluster 0 pgmap v157: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:17 smithi155 bash[59495]: cluster 2023-10-09T06:13:16.549880+0000 mgr.y (mgr.25723) 219 : cluster 0 pgmap v157: 73 pgs: 15 peering, 2 active+undersized+degraded, 9 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[77299]: audit 2023-10-09T06:13:18.188107+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.125:0/4231599176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[77299]: audit 2023-10-09T06:13:18.561783+0000 mon.a (mon.0) 275 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[77299]: audit 2023-10-09T06:13:18.563184+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[80002]: audit 2023-10-09T06:13:18.188107+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.125:0/4231599176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[80002]: audit 2023-10-09T06:13:18.561783+0000 mon.a (mon.0) 275 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:18.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:18 smithi125 bash[80002]: audit 2023-10-09T06:13:18.563184+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:19.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:18 smithi155 bash[59495]: audit 2023-10-09T06:13:18.188107+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.125:0/4231599176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:19.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:18 smithi155 bash[59495]: audit 2023-10-09T06:13:18.561783+0000 mon.a (mon.0) 275 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:19.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:18 smithi155 bash[59495]: audit 2023-10-09T06:13:18.563184+0000 mon.a (mon.0) 276 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:19.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[77299]: cluster 2023-10-09T06:13:18.550608+0000 mgr.y (mgr.25723) 220 : cluster 0 pgmap v158: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:19.908 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[77299]: cluster 2023-10-09T06:13:19.489432+0000 mon.a (mon.0) 277 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T06:13:19.908 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[77299]: cluster 2023-10-09T06:13:19.489503+0000 mon.a (mon.0) 278 : cluster 1 Cluster is now healthy 2023-10-09T06:13:19.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[80002]: cluster 2023-10-09T06:13:18.550608+0000 mgr.y (mgr.25723) 220 : cluster 0 pgmap v158: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:19.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[80002]: cluster 2023-10-09T06:13:19.489432+0000 mon.a (mon.0) 277 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T06:13:19.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:19 smithi125 bash[80002]: cluster 2023-10-09T06:13:19.489503+0000 mon.a (mon.0) 278 : cluster 1 Cluster is now healthy 2023-10-09T06:13:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:19 smithi155 bash[59495]: cluster 2023-10-09T06:13:18.550608+0000 mgr.y (mgr.25723) 220 : cluster 0 pgmap v158: 73 pgs: 15 peering, 58 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:19 smithi155 bash[59495]: cluster 2023-10-09T06:13:19.489432+0000 mon.a (mon.0) 277 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-09T06:13:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:19 smithi155 bash[59495]: cluster 2023-10-09T06:13:19.489503+0000 mon.a (mon.0) 278 : cluster 1 Cluster is now healthy 2023-10-09T06:13:20.947 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:20 smithi125 bash[77299]: audit 2023-10-09T06:13:20.547567+0000 mon.a (mon.0) 279 : audit 0 from='client.? 172.21.15.125:0/906845165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:20.948 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:20 smithi125 bash[80002]: audit 2023-10-09T06:13:20.547567+0000 mon.a (mon.0) 279 : audit 0 from='client.? 172.21.15.125:0/906845165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:21.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:20 smithi155 bash[59495]: audit 2023-10-09T06:13:20.547567+0000 mon.a (mon.0) 279 : audit 0 from='client.? 172.21.15.125:0/906845165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:21.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:21] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-10-09T06:13:22.025 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: cluster 2023-10-09T06:13:20.551568+0000 mgr.y (mgr.25723) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: audit 2023-10-09T06:13:21.598897+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: audit 2023-10-09T06:13:21.606717+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: audit 2023-10-09T06:13:21.608262+0000 mon.a (mon.0) 282 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: audit 2023-10-09T06:13:21.609494+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[77299]: audit 2023-10-09T06:13:21.617056+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: cluster 2023-10-09T06:13:20.551568+0000 mgr.y (mgr.25723) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: audit 2023-10-09T06:13:21.598897+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: audit 2023-10-09T06:13:21.606717+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: audit 2023-10-09T06:13:21.608262+0000 mon.a (mon.0) 282 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:22.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: audit 2023-10-09T06:13:21.609494+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:22.027 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:21 smithi125 bash[80002]: audit 2023-10-09T06:13:21.617056+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: cluster 2023-10-09T06:13:20.551568+0000 mgr.y (mgr.25723) 221 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: audit 2023-10-09T06:13:21.598897+0000 mon.a (mon.0) 280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: audit 2023-10-09T06:13:21.606717+0000 mon.a (mon.0) 281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: audit 2023-10-09T06:13:21.608262+0000 mon.a (mon.0) 282 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: audit 2023-10-09T06:13:21.609494+0000 mon.a (mon.0) 283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:21 smithi155 bash[59495]: audit 2023-10-09T06:13:21.617056+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:23.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.700086+0000 mon.a (mon.0) 285 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:23.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.704336+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.706747+0000 mon.a (mon.0) 287 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.708439+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.711725+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: audit 2023-10-09T06:13:21.712076+0000 mgr.y (mgr.25723) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[77299]: cephadm 2023-10-09T06:13:21.713602+0000 mgr.y (mgr.25723) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.700086+0000 mon.a (mon.0) 285 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.704336+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.706747+0000 mon.a (mon.0) 287 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.708439+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.711725+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: audit 2023-10-09T06:13:21.712076+0000 mgr.y (mgr.25723) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:22 smithi125 bash[80002]: cephadm 2023-10-09T06:13:21.713602+0000 mgr.y (mgr.25723) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.700086+0000 mon.a (mon.0) 285 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.704336+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.706747+0000 mon.a (mon.0) 287 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.708439+0000 mon.a (mon.0) 288 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.711725+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: audit 2023-10-09T06:13:21.712076+0000 mgr.y (mgr.25723) 222 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:13:23.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:22 smithi155 bash[59495]: cephadm 2023-10-09T06:13:21.713602+0000 mgr.y (mgr.25723) 223 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-09T06:13:23.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:23 smithi125 bash[77299]: cluster 2023-10-09T06:13:22.552012+0000 mgr.y (mgr.25723) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:23.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:23 smithi125 bash[77299]: audit 2023-10-09T06:13:22.907666+0000 mon.a (mon.0) 290 : audit 0 from='client.? 172.21.15.125:0/439389469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:23.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:23 smithi125 bash[80002]: cluster 2023-10-09T06:13:22.552012+0000 mgr.y (mgr.25723) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:23.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:23 smithi125 bash[80002]: audit 2023-10-09T06:13:22.907666+0000 mon.a (mon.0) 290 : audit 0 from='client.? 172.21.15.125:0/439389469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:23 smithi155 bash[59495]: cluster 2023-10-09T06:13:22.552012+0000 mgr.y (mgr.25723) 224 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:23 smithi155 bash[59495]: audit 2023-10-09T06:13:22.907666+0000 mon.a (mon.0) 290 : audit 0 from='client.? 172.21.15.125:0/439389469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: cephadm 2023-10-09T06:13:24.084516+0000 mgr.y (mgr.25723) 225 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: audit 2023-10-09T06:13:24.095470+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: audit 2023-10-09T06:13:24.097146+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: audit 2023-10-09T06:13:24.098389+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: cephadm 2023-10-09T06:13:24.100931+0000 mgr.y (mgr.25723) 226 : cephadm 1 Deploying daemon osd.1 on smithi125 2023-10-09T06:13:25.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[77299]: cluster 2023-10-09T06:13:24.553066+0000 mgr.y (mgr.25723) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: cephadm 2023-10-09T06:13:24.084516+0000 mgr.y (mgr.25723) 225 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: audit 2023-10-09T06:13:24.095470+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: audit 2023-10-09T06:13:24.097146+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: audit 2023-10-09T06:13:24.098389+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: cephadm 2023-10-09T06:13:24.100931+0000 mgr.y (mgr.25723) 226 : cephadm 1 Deploying daemon osd.1 on smithi125 2023-10-09T06:13:25.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:25 smithi125 bash[80002]: cluster 2023-10-09T06:13:24.553066+0000 mgr.y (mgr.25723) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: cephadm 2023-10-09T06:13:24.084516+0000 mgr.y (mgr.25723) 225 : cephadm 1 Upgrade: Updating osd.1 2023-10-09T06:13:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: audit 2023-10-09T06:13:24.095470+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:25.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: audit 2023-10-09T06:13:24.097146+0000 mon.a (mon.0) 292 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:13:25.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: audit 2023-10-09T06:13:24.098389+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:25.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: cephadm 2023-10-09T06:13:24.100931+0000 mgr.y (mgr.25723) 226 : cephadm 1 Deploying daemon osd.1 on smithi125 2023-10-09T06:13:25.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:25 smithi155 bash[59495]: cluster 2023-10-09T06:13:24.553066+0000 mgr.y (mgr.25723) 227 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:26.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:26 smithi125 bash[77299]: audit 2023-10-09T06:13:25.260893+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.125:0/1062793408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:26.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:26 smithi125 bash[80002]: audit 2023-10-09T06:13:25.260893+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.125:0/1062793408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:26 smithi155 bash[59495]: audit 2023-10-09T06:13:25.260893+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.125:0/1062793408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:27.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:27 smithi125 bash[77299]: cluster 2023-10-09T06:13:26.553734+0000 mgr.y (mgr.25723) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:27.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:27 smithi125 bash[80002]: cluster 2023-10-09T06:13:26.553734+0000 mgr.y (mgr.25723) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:27.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:27 smithi155 bash[59495]: cluster 2023-10-09T06:13:26.553734+0000 mgr.y (mgr.25723) 228 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:28 smithi125 bash[77299]: audit 2023-10-09T06:13:27.626534+0000 mon.a (mon.0) 294 : audit 0 from='client.? 172.21.15.125:0/906841610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:28.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:28 smithi125 bash[80002]: audit 2023-10-09T06:13:27.626534+0000 mon.a (mon.0) 294 : audit 0 from='client.? 172.21.15.125:0/906841610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:28 smithi155 bash[59495]: audit 2023-10-09T06:13:27.626534+0000 mon.a (mon.0) 294 : audit 0 from='client.? 172.21.15.125:0/906841610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:29.137 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:28 smithi125 systemd[1]: Stopping Ceph osd.1 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:13:29.627 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[41114]: debug 2023-10-09T06:13:29.135+0000 7f59d9ed4700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:13:29.628 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[41114]: debug 2023-10-09T06:13:29.135+0000 7f59d9ed4700 -1 osd.1 89 *** Got signal Terminated *** 2023-10-09T06:13:29.628 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[41114]: debug 2023-10-09T06:13:29.135+0000 7f59d9ed4700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:13:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[77299]: cluster 2023-10-09T06:13:28.554469+0000 mgr.y (mgr.25723) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[77299]: audit 2023-10-09T06:13:28.626924+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[77299]: cluster 2023-10-09T06:13:29.140230+0000 mon.a (mon.0) 296 : cluster 1 osd.1 marked itself down and dead 2023-10-09T06:13:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[80002]: cluster 2023-10-09T06:13:28.554469+0000 mgr.y (mgr.25723) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[80002]: audit 2023-10-09T06:13:28.626924+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:29 smithi125 bash[80002]: cluster 2023-10-09T06:13:29.140230+0000 mon.a (mon.0) 296 : cluster 1 osd.1 marked itself down and dead 2023-10-09T06:13:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:29 smithi155 bash[59495]: cluster 2023-10-09T06:13:28.554469+0000 mgr.y (mgr.25723) 229 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:29 smithi155 bash[59495]: audit 2023-10-09T06:13:28.626924+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:29 smithi155 bash[59495]: cluster 2023-10-09T06:13:29.140230+0000 mon.a (mon.0) 296 : cluster 1 osd.1 marked itself down and dead 2023-10-09T06:13:30.698 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[88052]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T06:13:30.698 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[88146]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[77299]: cluster 2023-10-09T06:13:29.621397+0000 mon.a (mon.0) 297 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[77299]: cluster 2023-10-09T06:13:29.636662+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[77299]: cluster 2023-10-09T06:13:29.638370+0000 mon.a (mon.0) 298 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[77299]: cluster 2023-10-09T06:13:29.638578+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[77299]: audit 2023-10-09T06:13:29.990486+0000 mon.a (mon.0) 299 : audit 0 from='client.? 172.21.15.125:0/1012368155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[80002]: cluster 2023-10-09T06:13:29.621397+0000 mon.a (mon.0) 297 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[80002]: cluster 2023-10-09T06:13:29.636662+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.699 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[80002]: cluster 2023-10-09T06:13:29.638370+0000 mon.a (mon.0) 298 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:13:30.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[80002]: cluster 2023-10-09T06:13:29.638578+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:30 smithi125 bash[80002]: audit 2023-10-09T06:13:29.990486+0000 mon.a (mon.0) 299 : audit 0 from='client.? 172.21.15.125:0/1012368155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:30 smithi155 bash[59495]: cluster 2023-10-09T06:13:29.621397+0000 mon.a (mon.0) 297 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:13:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:30 smithi155 bash[59495]: cluster 2023-10-09T06:13:29.636662+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:30 smithi155 bash[59495]: cluster 2023-10-09T06:13:29.638370+0000 mon.a (mon.0) 298 : cluster 0 osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:13:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:30 smithi155 bash[59495]: cluster 2023-10-09T06:13:29.638578+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e90 with expected crc 2023-10-09T06:13:30.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:30 smithi155 bash[59495]: audit 2023-10-09T06:13:29.990486+0000 mon.a (mon.0) 299 : audit 0 from='client.? 172.21.15.125:0/1012368155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:31.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:31] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.555248+0000 mgr.y (mgr.25723) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.643206+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.645141+0000 mon.a (mon.0) 300 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.645450+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.647843+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.647934+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.650445+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[80002]: cluster 2023-10-09T06:13:30.651611+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.555248+0000 mgr.y (mgr.25723) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.643206+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.645141+0000 mon.a (mon.0) 300 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.645450+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.647843+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.647934+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.650445+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:31 smithi125 bash[77299]: cluster 2023-10-09T06:13:30.651611+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.555248+0000 mgr.y (mgr.25723) 230 : cluster 0 pgmap v165: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.643206+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.645141+0000 mon.a (mon.0) 300 : cluster 0 osdmap e91: 8 total, 7 up, 8 in 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.645450+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.647843+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.647934+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.650445+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:31 smithi155 bash[59495]: cluster 2023-10-09T06:13:30.651611+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e91 with expected crc 2023-10-09T06:13:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:32 smithi125 bash[77299]: audit 2023-10-09T06:13:32.357333+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.125:0/4030749717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:32.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:32 smithi125 bash[80002]: audit 2023-10-09T06:13:32.357333+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.125:0/4030749717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:33.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:32 smithi155 bash[59495]: audit 2023-10-09T06:13:32.357333+0000 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.125:0/4030749717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:33.303 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:32 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T06:13:33.303 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:32 smithi125 systemd[1]: Stopped Ceph osd.1 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:13:33.303 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:33 smithi125 systemd[1]: Started Ceph osd.1 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:13:34.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: cluster 2023-10-09T06:13:32.555940+0000 mgr.y (mgr.25723) 231 : cluster 0 pgmap v167: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:34.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: audit 2023-10-09T06:13:33.266253+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: audit 2023-10-09T06:13:33.274321+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: audit 2023-10-09T06:13:33.284872+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: audit 2023-10-09T06:13:33.589403+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[77299]: audit 2023-10-09T06:13:33.590430+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: cluster 2023-10-09T06:13:32.555940+0000 mgr.y (mgr.25723) 231 : cluster 0 pgmap v167: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: audit 2023-10-09T06:13:33.266253+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: audit 2023-10-09T06:13:33.274321+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: audit 2023-10-09T06:13:33.284872+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: audit 2023-10-09T06:13:33.589403+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:33 smithi125 bash[80002]: audit 2023-10-09T06:13:33.590430+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: cluster 2023-10-09T06:13:32.555940+0000 mgr.y (mgr.25723) 231 : cluster 0 pgmap v167: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: audit 2023-10-09T06:13:33.266253+0000 mon.a (mon.0) 302 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: audit 2023-10-09T06:13:33.274321+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: audit 2023-10-09T06:13:33.284872+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: audit 2023-10-09T06:13:33.589403+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:33 smithi155 bash[59495]: audit 2023-10-09T06:13:33.590430+0000 mon.a (mon.0) 306 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:34.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[80002]: audit 2023-10-09T06:13:33.661795+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[80002]: audit 2023-10-09T06:13:33.667857+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[77299]: audit 2023-10-09T06:13:33.661795+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.919 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[77299]: audit 2023-10-09T06:13:33.667857+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:34.919 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T06:13:34.920 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:34 smithi125 bash[88332]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T06:13:35.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:34 smithi155 bash[59495]: audit 2023-10-09T06:13:33.661795+0000 mon.a (mon.0) 307 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:35.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:34 smithi155 bash[59495]: audit 2023-10-09T06:13:33.667857+0000 mon.a (mon.0) 308 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:36.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[80002]: cluster 2023-10-09T06:13:34.557090+0000 mgr.y (mgr.25723) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:36.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[80002]: cluster 2023-10-09T06:13:34.665597+0000 mon.a (mon.0) 309 : cluster 3 Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:36.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[80002]: audit 2023-10-09T06:13:34.720918+0000 mon.a (mon.0) 310 : audit 0 from='client.? 172.21.15.125:0/4137897520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:36.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[77299]: cluster 2023-10-09T06:13:34.557090+0000 mgr.y (mgr.25723) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:36.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[77299]: cluster 2023-10-09T06:13:34.665597+0000 mon.a (mon.0) 309 : cluster 3 Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:36.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:35 smithi125 bash[77299]: audit 2023-10-09T06:13:34.720918+0000 mon.a (mon.0) 310 : audit 0 from='client.? 172.21.15.125:0/4137897520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:35 smithi155 bash[59495]: cluster 2023-10-09T06:13:34.557090+0000 mgr.y (mgr.25723) 232 : cluster 0 pgmap v168: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:35 smithi155 bash[59495]: cluster 2023-10-09T06:13:34.665597+0000 mon.a (mon.0) 309 : cluster 3 Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:35 smithi155 bash[59495]: audit 2023-10-09T06:13:34.720918+0000 mon.a (mon.0) 310 : audit 0 from='client.? 172.21.15.125:0/4137897520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:37.137 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:36 smithi125 bash[88651]: debug 2023-10-09T06:13:36.803+0000 7f7935785700 1 -- 172.21.15.125:0/3589807813 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5617b0a444e0 con 0x5617b0a9ec00 2023-10-09T06:13:37.637 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:37 smithi125 bash[88651]: debug 2023-10-09T06:13:37.339+0000 7f793e90b640 -1 Falling back to public interface 2023-10-09T06:13:38.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:37 smithi125 bash[77299]: cluster 2023-10-09T06:13:36.557637+0000 mgr.y (mgr.25723) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:38.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:37 smithi125 bash[77299]: audit 2023-10-09T06:13:37.088936+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.125:0/204289984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:38.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:37 smithi125 bash[80002]: cluster 2023-10-09T06:13:36.557637+0000 mgr.y (mgr.25723) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:38.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:37 smithi125 bash[80002]: audit 2023-10-09T06:13:37.088936+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.125:0/204289984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:38.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:37 smithi155 bash[59495]: cluster 2023-10-09T06:13:36.557637+0000 mgr.y (mgr.25723) 233 : cluster 0 pgmap v169: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:38.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:37 smithi155 bash[59495]: audit 2023-10-09T06:13:37.088936+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.125:0/204289984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:39 smithi125 bash[77299]: cluster 2023-10-09T06:13:38.558485+0000 mgr.y (mgr.25723) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:39 smithi125 bash[77299]: audit 2023-10-09T06:13:39.472285+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.125:0/4052911421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:39 smithi125 bash[80002]: cluster 2023-10-09T06:13:38.558485+0000 mgr.y (mgr.25723) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:39 smithi125 bash[80002]: audit 2023-10-09T06:13:39.472285+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.125:0/4052911421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:39 smithi155 bash[59495]: cluster 2023-10-09T06:13:38.558485+0000 mgr.y (mgr.25723) 234 : cluster 0 pgmap v170: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:39 smithi155 bash[59495]: audit 2023-10-09T06:13:39.472285+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.125:0/4052911421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:41.137 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:40 smithi125 bash[88651]: debug 2023-10-09T06:13:40.875+0000 7f793e90b640 -1 osd.1 89 log_to_monitors true 2023-10-09T06:13:41.637 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[88651]: debug 2023-10-09T06:13:41.439+0000 7f792d572700 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:13:41.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:41] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.33.4" 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:40.381111+0000 mon.a (mon.0) 312 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:40.389382+0000 mon.a (mon.0) 313 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: cluster 2023-10-09T06:13:40.559705+0000 mgr.y (mgr.25723) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:40.884146+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:40.884516+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:41.184417+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[77299]: audit 2023-10-09T06:13:41.194846+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:40.381111+0000 mon.a (mon.0) 312 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:40.389382+0000 mon.a (mon.0) 313 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: cluster 2023-10-09T06:13:40.559705+0000 mgr.y (mgr.25723) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:40.884146+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:40.884516+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:41.184417+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:41 smithi125 bash[80002]: audit 2023-10-09T06:13:41.194846+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:40.381111+0000 mon.a (mon.0) 312 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:40.389382+0000 mon.a (mon.0) 313 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: cluster 2023-10-09T06:13:40.559705+0000 mgr.y (mgr.25723) 235 : cluster 0 pgmap v171: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:40.884146+0000 mon.c (mon.2) 16 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:40.884516+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:41.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:41.184417+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:41 smithi155 bash[59495]: audit 2023-10-09T06:13:41.194846+0000 mon.a (mon.0) 316 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:42.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: cluster 2023-10-09T06:13:41.395024+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: audit 2023-10-09T06:13:41.396726+0000 mon.a (mon.0) 317 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:42.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: cluster 2023-10-09T06:13:41.396804+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: cluster 2023-10-09T06:13:41.396829+0000 mon.a (mon.0) 318 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: audit 2023-10-09T06:13:41.397795+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: audit 2023-10-09T06:13:41.405244+0000 mon.a (mon.0) 319 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[77299]: audit 2023-10-09T06:13:41.830198+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.125:0/1323268183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: cluster 2023-10-09T06:13:41.395024+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: audit 2023-10-09T06:13:41.396726+0000 mon.a (mon.0) 317 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: cluster 2023-10-09T06:13:41.396804+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: cluster 2023-10-09T06:13:41.396829+0000 mon.a (mon.0) 318 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T06:13:42.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: audit 2023-10-09T06:13:41.397795+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: audit 2023-10-09T06:13:41.405244+0000 mon.a (mon.0) 319 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:42 smithi125 bash[80002]: audit 2023-10-09T06:13:41.830198+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.125:0/1323268183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: cluster 2023-10-09T06:13:41.395024+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: audit 2023-10-09T06:13:41.396726+0000 mon.a (mon.0) 317 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: cluster 2023-10-09T06:13:41.396804+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:42.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: cluster 2023-10-09T06:13:41.396829+0000 mon.a (mon.0) 318 : cluster 0 osdmap e92: 8 total, 7 up, 8 in 2023-10-09T06:13:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: audit 2023-10-09T06:13:41.397795+0000 mon.c (mon.2) 17 : audit 1 from='osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: audit 2023-10-09T06:13:41.405244+0000 mon.a (mon.0) 319 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:13:42.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:42 smithi155 bash[59495]: audit 2023-10-09T06:13:41.830198+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.125:0/1323268183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.400557+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:43.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.403248+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.405622+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262] boot 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.405687+0000 mon.a (mon.0) 322 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.406134+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: audit 2023-10-09T06:13:42.406523+0000 mon.a (mon.0) 323 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[77299]: cluster 2023-10-09T06:13:42.560542+0000 mgr.y (mgr.25723) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.400557+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.403248+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.405622+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262] boot 2023-10-09T06:13:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.405687+0000 mon.a (mon.0) 322 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:13:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.406134+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: audit 2023-10-09T06:13:42.406523+0000 mon.a (mon.0) 323 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:43.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:43 smithi125 bash[80002]: cluster 2023-10-09T06:13:42.560542+0000 mgr.y (mgr.25723) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.400557+0000 mon.a (mon.0) 320 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.403248+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.405622+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262] boot 2023-10-09T06:13:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.405687+0000 mon.a (mon.0) 322 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:13:43.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.406134+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e92 with expected crc 2023-10-09T06:13:43.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: audit 2023-10-09T06:13:42.406523+0000 mon.a (mon.0) 323 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:43.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:43 smithi155 bash[59495]: cluster 2023-10-09T06:13:42.560542+0000 mgr.y (mgr.25723) 236 : cluster 0 pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.417910+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.418660+0000 mon.a (mon.0) 324 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.419841+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.420103+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.420655+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.421366+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.421889+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.422000+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.423879+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.424134+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.424935+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.425089+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.425184+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.425976+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: cluster 2023-10-09T06:13:43.426342+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[77299]: audit 2023-10-09T06:13:44.197875+0000 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.125:0/2977910688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.417910+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.418660+0000 mon.a (mon.0) 324 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.419841+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.420103+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.420655+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.421366+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.421889+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.422000+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.423879+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.424134+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.424935+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.425089+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.425184+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.425976+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: cluster 2023-10-09T06:13:43.426342+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:44 smithi125 bash[80002]: audit 2023-10-09T06:13:44.197875+0000 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.125:0/2977910688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.417910+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.418660+0000 mon.a (mon.0) 324 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:13:44.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.419841+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.420103+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.420655+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.421366+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.421889+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.422000+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.423879+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.424134+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.424935+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.425089+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.425184+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.425976+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: cluster 2023-10-09T06:13:43.426342+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e94 with expected crc 2023-10-09T06:13:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:44 smithi155 bash[59495]: audit 2023-10-09T06:13:44.197875+0000 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.125:0/2977910688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:45.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:45 smithi125 bash[77299]: cluster 2023-10-09T06:13:44.561523+0000 mgr.y (mgr.25723) 237 : cluster 0 pgmap v176: 73 pgs: 3 peering, 1 active+undersized+degraded, 9 active+undersized, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:13:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:45 smithi125 bash[80002]: cluster 2023-10-09T06:13:44.561523+0000 mgr.y (mgr.25723) 237 : cluster 0 pgmap v176: 73 pgs: 3 peering, 1 active+undersized+degraded, 9 active+undersized, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:13:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:45 smithi155 bash[59495]: cluster 2023-10-09T06:13:44.561523+0000 mgr.y (mgr.25723) 237 : cluster 0 pgmap v176: 73 pgs: 3 peering, 1 active+undersized+degraded, 9 active+undersized, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:13:46.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:46 smithi125 bash[77299]: cluster 2023-10-09T06:13:45.412248+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:13:46.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:46 smithi125 bash[80002]: cluster 2023-10-09T06:13:45.412248+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:13:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:46 smithi155 bash[59495]: cluster 2023-10-09T06:13:45.412248+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:13:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:47 smithi125 bash[77299]: audit 2023-10-09T06:13:46.553351+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.125:0/1689314484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:47 smithi125 bash[77299]: cluster 2023-10-09T06:13:46.562156+0000 mgr.y (mgr.25723) 238 : cluster 0 pgmap v177: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:47.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:47 smithi125 bash[80002]: audit 2023-10-09T06:13:46.553351+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.125:0/1689314484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:47.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:47 smithi125 bash[80002]: cluster 2023-10-09T06:13:46.562156+0000 mgr.y (mgr.25723) 238 : cluster 0 pgmap v177: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:47 smithi155 bash[59495]: audit 2023-10-09T06:13:46.553351+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.125:0/1689314484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:47.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:47 smithi155 bash[59495]: cluster 2023-10-09T06:13:46.562156+0000 mgr.y (mgr.25723) 238 : cluster 0 pgmap v177: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:48.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:48 smithi125 bash[80002]: cluster 2023-10-09T06:13:47.422614+0000 mon.a (mon.0) 327 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:13:48.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:48 smithi125 bash[80002]: cluster 2023-10-09T06:13:47.422671+0000 mon.a (mon.0) 328 : cluster 1 Cluster is now healthy 2023-10-09T06:13:48.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:48 smithi125 bash[77299]: cluster 2023-10-09T06:13:47.422614+0000 mon.a (mon.0) 327 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:13:48.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:48 smithi125 bash[77299]: cluster 2023-10-09T06:13:47.422671+0000 mon.a (mon.0) 328 : cluster 1 Cluster is now healthy 2023-10-09T06:13:48.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:48 smithi155 bash[59495]: cluster 2023-10-09T06:13:47.422614+0000 mon.a (mon.0) 327 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:13:48.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:48 smithi155 bash[59495]: cluster 2023-10-09T06:13:47.422671+0000 mon.a (mon.0) 328 : cluster 1 Cluster is now healthy 2023-10-09T06:13:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[77299]: audit 2023-10-09T06:13:48.561090+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[77299]: audit 2023-10-09T06:13:48.562484+0000 mon.a (mon.0) 330 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[77299]: cluster 2023-10-09T06:13:48.562821+0000 mgr.y (mgr.25723) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[77299]: audit 2023-10-09T06:13:48.904350+0000 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.125:0/3529937490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[80002]: audit 2023-10-09T06:13:48.561090+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[80002]: audit 2023-10-09T06:13:48.562484+0000 mon.a (mon.0) 330 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[80002]: cluster 2023-10-09T06:13:48.562821+0000 mgr.y (mgr.25723) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:49 smithi125 bash[80002]: audit 2023-10-09T06:13:48.904350+0000 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.125:0/3529937490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:49 smithi155 bash[59495]: audit 2023-10-09T06:13:48.561090+0000 mon.a (mon.0) 329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:49 smithi155 bash[59495]: audit 2023-10-09T06:13:48.562484+0000 mon.a (mon.0) 330 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:49 smithi155 bash[59495]: cluster 2023-10-09T06:13:48.562821+0000 mgr.y (mgr.25723) 239 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:49 smithi155 bash[59495]: audit 2023-10-09T06:13:48.904350+0000 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.125:0/3529937490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:51.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:13:51] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.33.4" 2023-10-09T06:13:52.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: cluster 2023-10-09T06:13:50.563732+0000 mgr.y (mgr.25723) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:52.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.896068+0000 mon.a (mon.0) 332 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.903017+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.904392+0000 mon.a (mon.0) 334 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:52.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.905574+0000 mon.a (mon.0) 335 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:52.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.912105+0000 mon.a (mon.0) 336 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.270 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:50.996642+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:52.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:51.001878+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:51.005307+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:51.006741+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:51.008475+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:52.271 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[77299]: audit 2023-10-09T06:13:51.254575+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.125:0/4244948048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:52.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: cluster 2023-10-09T06:13:50.563732+0000 mgr.y (mgr.25723) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:52.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.896068+0000 mon.a (mon.0) 332 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.903017+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.904392+0000 mon.a (mon.0) 334 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:52.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.905574+0000 mon.a (mon.0) 335 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.912105+0000 mon.a (mon.0) 336 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:50.996642+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:51.001878+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:51.005307+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:51.006741+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:51.008475+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:52.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:51 smithi125 bash[80002]: audit 2023-10-09T06:13:51.254575+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.125:0/4244948048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: cluster 2023-10-09T06:13:50.563732+0000 mgr.y (mgr.25723) 240 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.896068+0000 mon.a (mon.0) 332 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.903017+0000 mon.a (mon.0) 333 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.904392+0000 mon.a (mon.0) 334 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.905574+0000 mon.a (mon.0) 335 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.912105+0000 mon.a (mon.0) 336 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:50.996642+0000 mon.a (mon.0) 337 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:51.001878+0000 mon.a (mon.0) 338 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:51.005307+0000 mon.a (mon.0) 339 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:51.006741+0000 mon.a (mon.0) 340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:52.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:51.008475+0000 mon.a (mon.0) 341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:52.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:51 smithi155 bash[59495]: audit 2023-10-09T06:13:51.254575+0000 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.125:0/4244948048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:53.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:52 smithi125 bash[80002]: audit 2023-10-09T06:13:51.008671+0000 mgr.y (mgr.25723) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:53.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:52 smithi125 bash[80002]: cephadm 2023-10-09T06:13:51.009659+0000 mgr.y (mgr.25723) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T06:13:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:52 smithi125 bash[77299]: audit 2023-10-09T06:13:51.008671+0000 mgr.y (mgr.25723) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:53.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:52 smithi125 bash[77299]: cephadm 2023-10-09T06:13:51.009659+0000 mgr.y (mgr.25723) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T06:13:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:52 smithi155 bash[59495]: audit 2023-10-09T06:13:51.008671+0000 mgr.y (mgr.25723) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:52 smithi155 bash[59495]: cephadm 2023-10-09T06:13:51.009659+0000 mgr.y (mgr.25723) 242 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-09T06:13:54.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: cluster 2023-10-09T06:13:52.564338+0000 mgr.y (mgr.25723) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:54.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: audit 2023-10-09T06:13:53.337255+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: audit 2023-10-09T06:13:53.338711+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: audit 2023-10-09T06:13:53.339593+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: audit 2023-10-09T06:13:53.621719+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.125:0/1820468860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[77299]: audit 2023-10-09T06:13:53.644701+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: cluster 2023-10-09T06:13:52.564338+0000 mgr.y (mgr.25723) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: audit 2023-10-09T06:13:53.337255+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:54.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: audit 2023-10-09T06:13:53.338711+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:13:54.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: audit 2023-10-09T06:13:53.339593+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:54.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: audit 2023-10-09T06:13:53.621719+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.125:0/1820468860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:54.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:53 smithi125 bash[80002]: audit 2023-10-09T06:13:53.644701+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: cluster 2023-10-09T06:13:52.564338+0000 mgr.y (mgr.25723) 243 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: audit 2023-10-09T06:13:53.337255+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: audit 2023-10-09T06:13:53.338711+0000 mon.a (mon.0) 344 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: audit 2023-10-09T06:13:53.339593+0000 mon.a (mon.0) 345 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: audit 2023-10-09T06:13:53.621719+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.125:0/1820468860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:54.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:53 smithi155 bash[59495]: audit 2023-10-09T06:13:53.644701+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:13:55.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:54 smithi125 bash[80002]: cephadm 2023-10-09T06:13:53.328479+0000 mgr.y (mgr.25723) 244 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T06:13:55.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:54 smithi125 bash[80002]: cephadm 2023-10-09T06:13:53.342815+0000 mgr.y (mgr.25723) 245 : cephadm 1 Deploying daemon osd.2 on smithi125 2023-10-09T06:13:55.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:54 smithi125 bash[77299]: cephadm 2023-10-09T06:13:53.328479+0000 mgr.y (mgr.25723) 244 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T06:13:55.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:54 smithi125 bash[77299]: cephadm 2023-10-09T06:13:53.342815+0000 mgr.y (mgr.25723) 245 : cephadm 1 Deploying daemon osd.2 on smithi125 2023-10-09T06:13:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:54 smithi155 bash[59495]: cephadm 2023-10-09T06:13:53.328479+0000 mgr.y (mgr.25723) 244 : cephadm 1 Upgrade: Updating osd.2 2023-10-09T06:13:55.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:54 smithi155 bash[59495]: cephadm 2023-10-09T06:13:53.342815+0000 mgr.y (mgr.25723) 245 : cephadm 1 Deploying daemon osd.2 on smithi125 2023-10-09T06:13:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:55 smithi125 bash[77299]: cluster 2023-10-09T06:13:54.564960+0000 mgr.y (mgr.25723) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:55 smithi125 bash[80002]: cluster 2023-10-09T06:13:54.564960+0000 mgr.y (mgr.25723) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:55 smithi155 bash[59495]: cluster 2023-10-09T06:13:54.564960+0000 mgr.y (mgr.25723) 246 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:57.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:56 smithi125 bash[77299]: audit 2023-10-09T06:13:55.981184+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.125:0/2935832736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:57.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:56 smithi125 bash[80002]: audit 2023-10-09T06:13:55.981184+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.125:0/2935832736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:57.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:56 smithi155 bash[59495]: audit 2023-10-09T06:13:55.981184+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.125:0/2935832736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:58.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:57 smithi125 bash[77299]: cluster 2023-10-09T06:13:56.565448+0000 mgr.y (mgr.25723) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:58.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:57 smithi125 bash[80002]: cluster 2023-10-09T06:13:56.565448+0000 mgr.y (mgr.25723) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:57 smithi155 bash[59495]: cluster 2023-10-09T06:13:56.565448+0000 mgr.y (mgr.25723) 247 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:13:58.637 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:58 smithi125 systemd[1]: Stopping Ceph osd.2 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:13:58.637 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[45304]: debug 2023-10-09T06:13:58.326+0000 7f715e154700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:13:58.638 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[45304]: debug 2023-10-09T06:13:58.326+0000 7f715e154700 -1 osd.2 94 *** Got signal Terminated *** 2023-10-09T06:13:58.638 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[45304]: debug 2023-10-09T06:13:58.326+0000 7f715e154700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:13:59.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.332571+0000 mon.a (mon.0) 347 : cluster 1 osd.2 marked itself down and dead 2023-10-09T06:13:59.241 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[77299]: audit 2023-10-09T06:13:58.356763+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.125:0/570464808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:59.241 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.332571+0000 mon.a (mon.0) 347 : cluster 1 osd.2 marked itself down and dead 2023-10-09T06:13:59.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:58 smithi125 bash[80002]: audit 2023-10-09T06:13:58.356763+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.125:0/570464808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:58 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.332571+0000 mon.a (mon.0) 347 : cluster 1 osd.2 marked itself down and dead 2023-10-09T06:13:59.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:58 smithi155 bash[59495]: audit 2023-10-09T06:13:58.356763+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.125:0/570464808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:59.943 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[90930]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T06:13:59.944 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[90998]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T06:14:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.566152+0000 mgr.y (mgr.25723) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:00.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.934423+0000 mon.a (mon.0) 348 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.949539+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.951862+0000 mon.a (mon.0) 349 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.957924+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[80002]: cluster 2023-10-09T06:13:58.958128+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.566152+0000 mgr.y (mgr.25723) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.934423+0000 mon.a (mon.0) 348 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.949539+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.951862+0000 mon.a (mon.0) 349 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T06:14:00.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.957924+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:13:59 smithi125 bash[77299]: cluster 2023-10-09T06:13:58.958128+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.566152+0000 mgr.y (mgr.25723) 248 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.934423+0000 mon.a (mon.0) 348 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.949539+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.951862+0000 mon.a (mon.0) 349 : cluster 0 osdmap e95: 8 total, 7 up, 8 in 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.957924+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:13:59 smithi155 bash[59495]: cluster 2023-10-09T06:13:58.958128+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e95 with expected crc 2023-10-09T06:14:01.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.950000+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.953510+0000 mon.a (mon.0) 350 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.954415+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.954646+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.955844+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.956327+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.956415+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.956649+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.957230+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.957583+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: cluster 2023-10-09T06:13:59.961969+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[80002]: audit 2023-10-09T06:14:00.720939+0000 mon.a (mon.0) 351 : audit 0 from='client.? 172.21.15.125:0/2259025764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.950000+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.953510+0000 mon.a (mon.0) 350 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.954415+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.954646+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.955844+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.956327+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.956415+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.956649+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.957230+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.957583+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: cluster 2023-10-09T06:13:59.961969+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:00 smithi125 bash[77299]: audit 2023-10-09T06:14:00.720939+0000 mon.a (mon.0) 351 : audit 0 from='client.? 172.21.15.125:0/2259025764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.950000+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.953510+0000 mon.a (mon.0) 350 : cluster 0 osdmap e96: 8 total, 7 up, 8 in 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.954415+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.954646+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.955844+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.956327+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.956415+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.956649+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.957230+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.957583+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: cluster 2023-10-09T06:13:59.961969+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e96 with expected crc 2023-10-09T06:14:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:00 smithi155 bash[59495]: audit 2023-10-09T06:14:00.720939+0000 mon.a (mon.0) 351 : audit 0 from='client.? 172.21.15.125:0/2259025764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:01.504 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:01] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.33.4" 2023-10-09T06:14:02.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:01 smithi125 bash[77299]: cluster 2023-10-09T06:14:00.566612+0000 mgr.y (mgr.25723) 249 : cluster 0 pgmap v186: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:02.249 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:01 smithi125 bash[77299]: cluster 2023-10-09T06:14:00.948081+0000 mon.a (mon.0) 352 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:02.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:01 smithi125 bash[80002]: cluster 2023-10-09T06:14:00.566612+0000 mgr.y (mgr.25723) 249 : cluster 0 pgmap v186: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:02.250 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:01 smithi125 bash[80002]: cluster 2023-10-09T06:14:00.948081+0000 mon.a (mon.0) 352 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:01 smithi155 bash[59495]: cluster 2023-10-09T06:14:00.566612+0000 mgr.y (mgr.25723) 249 : cluster 0 pgmap v186: 73 pgs: 13 peering, 60 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:01 smithi155 bash[59495]: cluster 2023-10-09T06:14:00.948081+0000 mon.a (mon.0) 352 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:02.509 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:02 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T06:14:02.509 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:02 smithi125 systemd[1]: Stopped Ceph osd.2 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:02.510 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:02 smithi125 systemd[1]: Started Ceph osd.2 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[77299]: audit 2023-10-09T06:14:02.544502+0000 mon.a (mon.0) 353 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[77299]: audit 2023-10-09T06:14:02.553431+0000 mon.a (mon.0) 354 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[77299]: cluster 2023-10-09T06:14:02.567131+0000 mgr.y (mgr.25723) 250 : cluster 0 pgmap v187: 73 pgs: 13 peering, 4 active+undersized, 56 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[77299]: audit 2023-10-09T06:14:02.577648+0000 mon.a (mon.0) 355 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[77299]: audit 2023-10-09T06:14:03.069853+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.125:0/1414101725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[80002]: audit 2023-10-09T06:14:02.544502+0000 mon.a (mon.0) 353 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[80002]: audit 2023-10-09T06:14:02.553431+0000 mon.a (mon.0) 354 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[80002]: cluster 2023-10-09T06:14:02.567131+0000 mgr.y (mgr.25723) 250 : cluster 0 pgmap v187: 73 pgs: 13 peering, 4 active+undersized, 56 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[80002]: audit 2023-10-09T06:14:02.577648+0000 mon.a (mon.0) 355 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:03.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:03 smithi125 bash[80002]: audit 2023-10-09T06:14:03.069853+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.125:0/1414101725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:03 smithi155 bash[59495]: audit 2023-10-09T06:14:02.544502+0000 mon.a (mon.0) 353 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:03 smithi155 bash[59495]: audit 2023-10-09T06:14:02.553431+0000 mon.a (mon.0) 354 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:03 smithi155 bash[59495]: cluster 2023-10-09T06:14:02.567131+0000 mgr.y (mgr.25723) 250 : cluster 0 pgmap v187: 73 pgs: 13 peering, 4 active+undersized, 56 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:14:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:03 smithi155 bash[59495]: audit 2023-10-09T06:14:02.577648+0000 mon.a (mon.0) 355 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:03 smithi155 bash[59495]: audit 2023-10-09T06:14:03.069853+0000 mon.a (mon.0) 356 : audit 0 from='client.? 172.21.15.125:0/1414101725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:04.515 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:14:04.515 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T06:14:04.516 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[91179]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T06:14:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[77299]: audit 2023-10-09T06:14:03.558165+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[77299]: audit 2023-10-09T06:14:03.559345+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[80002]: audit 2023-10-09T06:14:03.558165+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:04 smithi125 bash[80002]: audit 2023-10-09T06:14:03.559345+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:04 smithi155 bash[59495]: audit 2023-10-09T06:14:03.558165+0000 mon.a (mon.0) 357 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:04 smithi155 bash[59495]: audit 2023-10-09T06:14:03.559345+0000 mon.a (mon.0) 358 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:05.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:05 smithi125 bash[77299]: cluster 2023-10-09T06:14:04.567721+0000 mgr.y (mgr.25723) 251 : cluster 0 pgmap v188: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:05.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:05 smithi125 bash[77299]: audit 2023-10-09T06:14:05.424496+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.125:0/1982122337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:05.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:05 smithi125 bash[80002]: cluster 2023-10-09T06:14:04.567721+0000 mgr.y (mgr.25723) 251 : cluster 0 pgmap v188: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:05.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:05 smithi125 bash[80002]: audit 2023-10-09T06:14:05.424496+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.125:0/1982122337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:05 smithi155 bash[59495]: cluster 2023-10-09T06:14:04.567721+0000 mgr.y (mgr.25723) 251 : cluster 0 pgmap v188: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:05 smithi155 bash[59495]: audit 2023-10-09T06:14:05.424496+0000 mon.a (mon.0) 359 : audit 0 from='client.? 172.21.15.125:0/1982122337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:06.387 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[91497]: debug 2023-10-09T06:14:06.090+0000 7f4127813700 1 -- 172.21.15.125:0/2475666718 <== mon.2 v2:172.21.15.125:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555dd516e4e0 con 0x555dd51c8c00 2023-10-09T06:14:06.887 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[91497]: debug 2023-10-09T06:14:06.654+0000 7f4130999640 -1 Falling back to public interface 2023-10-09T06:14:06.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[77299]: cluster 2023-10-09T06:14:05.558341+0000 mon.a (mon.0) 360 : cluster 3 Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:06.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[77299]: cluster 2023-10-09T06:14:05.558402+0000 mon.a (mon.0) 361 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:14:06.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[80002]: cluster 2023-10-09T06:14:05.558341+0000 mon.a (mon.0) 360 : cluster 3 Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:06.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:06 smithi125 bash[80002]: cluster 2023-10-09T06:14:05.558402+0000 mon.a (mon.0) 361 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:14:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:06 smithi155 bash[59495]: cluster 2023-10-09T06:14:05.558341+0000 mon.a (mon.0) 360 : cluster 3 Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:06.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:06 smithi155 bash[59495]: cluster 2023-10-09T06:14:05.558402+0000 mon.a (mon.0) 361 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:14:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:07 smithi125 bash[77299]: cluster 2023-10-09T06:14:06.568502+0000 mgr.y (mgr.25723) 252 : cluster 0 pgmap v189: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:07 smithi125 bash[80002]: cluster 2023-10-09T06:14:06.568502+0000 mgr.y (mgr.25723) 252 : cluster 0 pgmap v189: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:07 smithi155 bash[59495]: cluster 2023-10-09T06:14:06.568502+0000 mgr.y (mgr.25723) 252 : cluster 0 pgmap v189: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:08.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:08 smithi125 bash[77299]: audit 2023-10-09T06:14:07.784934+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.125:0/2070711628' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:08.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:08 smithi125 bash[80002]: audit 2023-10-09T06:14:07.784934+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.125:0/2070711628' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:08.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:08 smithi155 bash[59495]: audit 2023-10-09T06:14:07.784934+0000 mon.a (mon.0) 362 : audit 0 from='client.? 172.21.15.125:0/2070711628' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:10.137 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[91497]: debug 2023-10-09T06:14:09.926+0000 7f4130999640 -1 osd.2 94 log_to_monitors true 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[77299]: cluster 2023-10-09T06:14:08.569358+0000 mgr.y (mgr.25723) 253 : cluster 0 pgmap v190: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[77299]: audit 2023-10-09T06:14:08.832738+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[77299]: audit 2023-10-09T06:14:08.842125+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[77299]: audit 2023-10-09T06:14:09.792287+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[77299]: audit 2023-10-09T06:14:09.800654+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[80002]: cluster 2023-10-09T06:14:08.569358+0000 mgr.y (mgr.25723) 253 : cluster 0 pgmap v190: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[80002]: audit 2023-10-09T06:14:08.832738+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[80002]: audit 2023-10-09T06:14:08.842125+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[80002]: audit 2023-10-09T06:14:09.792287+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:09 smithi125 bash[80002]: audit 2023-10-09T06:14:09.800654+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:09 smithi155 bash[59495]: cluster 2023-10-09T06:14:08.569358+0000 mgr.y (mgr.25723) 253 : cluster 0 pgmap v190: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:09 smithi155 bash[59495]: audit 2023-10-09T06:14:08.832738+0000 mon.a (mon.0) 363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:09 smithi155 bash[59495]: audit 2023-10-09T06:14:08.842125+0000 mon.a (mon.0) 364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:09 smithi155 bash[59495]: audit 2023-10-09T06:14:09.792287+0000 mon.a (mon.0) 365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:09 smithi155 bash[59495]: audit 2023-10-09T06:14:09.800654+0000 mon.a (mon.0) 366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.137 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[91497]: debug 2023-10-09T06:14:10.886+0000 7f411edff700 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[77299]: audit 2023-10-09T06:14:09.936116+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[77299]: audit 2023-10-09T06:14:10.148915+0000 mon.a (mon.0) 368 : audit 0 from='client.? 172.21.15.125:0/3497840193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[77299]: audit 2023-10-09T06:14:10.645788+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[77299]: audit 2023-10-09T06:14:10.654972+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[80002]: audit 2023-10-09T06:14:09.936116+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[80002]: audit 2023-10-09T06:14:10.148915+0000 mon.a (mon.0) 368 : audit 0 from='client.? 172.21.15.125:0/3497840193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[80002]: audit 2023-10-09T06:14:10.645788+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:10 smithi125 bash[80002]: audit 2023-10-09T06:14:10.654972+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:10 smithi155 bash[59495]: audit 2023-10-09T06:14:09.936116+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:14:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:10 smithi155 bash[59495]: audit 2023-10-09T06:14:10.148915+0000 mon.a (mon.0) 368 : audit 0 from='client.? 172.21.15.125:0/3497840193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:10 smithi155 bash[59495]: audit 2023-10-09T06:14:10.645788+0000 mon.a (mon.0) 369 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:10 smithi155 bash[59495]: audit 2023-10-09T06:14:10.654972+0000 mon.a (mon.0) 370 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:11.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:11] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.33.4" 2023-10-09T06:14:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.570344+0000 mgr.y (mgr.25723) 254 : cluster 0 pgmap v191: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: audit 2023-10-09T06:14:10.843797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:14:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.843869+0000 mon.a (mon.0) 372 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T06:14:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: audit 2023-10-09T06:14:10.844434+0000 mon.a (mon.0) 373 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.844712+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.871908+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.952886+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.955648+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[77299]: cluster 2023-10-09T06:14:10.955729+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.570344+0000 mgr.y (mgr.25723) 254 : cluster 0 pgmap v191: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: audit 2023-10-09T06:14:10.843797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.843869+0000 mon.a (mon.0) 372 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: audit 2023-10-09T06:14:10.844434+0000 mon.a (mon.0) 373 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.844712+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.871908+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.952886+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.955648+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:11 smithi125 bash[80002]: cluster 2023-10-09T06:14:10.955729+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.570344+0000 mgr.y (mgr.25723) 254 : cluster 0 pgmap v191: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: audit 2023-10-09T06:14:10.843797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.843869+0000 mon.a (mon.0) 372 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: audit 2023-10-09T06:14:10.844434+0000 mon.a (mon.0) 373 : audit 1 from='osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.844712+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.871908+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.952886+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.955648+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:12.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:11 smithi155 bash[59495]: cluster 2023-10-09T06:14:10.955729+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e97 with expected crc 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.841125+0000 mon.a (mon.0) 374 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.844712+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.848004+0000 mon.a (mon.0) 375 : cluster 1 osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518] boot 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.848087+0000 mon.a (mon.0) 376 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: audit 2023-10-09T06:14:11.848974+0000 mon.a (mon.0) 377 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.853736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.855452+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: cluster 2023-10-09T06:14:11.855527+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[77299]: audit 2023-10-09T06:14:12.513712+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.125:0/4251227598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.841125+0000 mon.a (mon.0) 374 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.844712+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.848004+0000 mon.a (mon.0) 375 : cluster 1 osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518] boot 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.848087+0000 mon.a (mon.0) 376 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: audit 2023-10-09T06:14:11.848974+0000 mon.a (mon.0) 377 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.853736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.855452+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: cluster 2023-10-09T06:14:11.855527+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:12 smithi125 bash[80002]: audit 2023-10-09T06:14:12.513712+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.125:0/4251227598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:13.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.841125+0000 mon.a (mon.0) 374 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:13.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.844712+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.848004+0000 mon.a (mon.0) 375 : cluster 1 osd.2 [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518] boot 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.848087+0000 mon.a (mon.0) 376 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: audit 2023-10-09T06:14:11.848974+0000 mon.a (mon.0) 377 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.853736+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.855452+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: cluster 2023-10-09T06:14:11.855527+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e98 with expected crc 2023-10-09T06:14:13.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:12 smithi155 bash[59495]: audit 2023-10-09T06:14:12.513712+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.125:0/4251227598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.571059+0000 mgr.y (mgr.25723) 255 : cluster 0 pgmap v194: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.870680+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.875471+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.875927+0000 mon.a (mon.0) 378 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.876201+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.878823+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.878931+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.878936+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.881317+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.883167+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.883579+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: cluster 2023-10-09T06:14:12.885651+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[77299]: audit 2023-10-09T06:14:13.663149+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.571059+0000 mgr.y (mgr.25723) 255 : cluster 0 pgmap v194: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.870680+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.875471+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.875927+0000 mon.a (mon.0) 378 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.876201+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.878823+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.878931+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.878936+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.881317+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.883167+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.883579+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: cluster 2023-10-09T06:14:12.885651+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:13 smithi125 bash[80002]: audit 2023-10-09T06:14:13.663149+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.571059+0000 mgr.y (mgr.25723) 255 : cluster 0 pgmap v194: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.870680+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.875471+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.875927+0000 mon.a (mon.0) 378 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.876201+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.878823+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.878931+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.878936+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.881317+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.883167+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.883579+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: cluster 2023-10-09T06:14:12.885651+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e99 with expected crc 2023-10-09T06:14:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:13 smithi155 bash[59495]: audit 2023-10-09T06:14:13.663149+0000 mon.a (mon.0) 379 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:15.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:14 smithi125 bash[77299]: cluster 2023-10-09T06:14:14.869063+0000 mon.a (mon.0) 380 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:15.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:14 smithi125 bash[77299]: audit 2023-10-09T06:14:14.875059+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.125:0/2311714980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:15.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:14 smithi125 bash[80002]: cluster 2023-10-09T06:14:14.869063+0000 mon.a (mon.0) 380 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:15.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:14 smithi125 bash[80002]: audit 2023-10-09T06:14:14.875059+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.125:0/2311714980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:15.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:14 smithi155 bash[59495]: cluster 2023-10-09T06:14:14.869063+0000 mon.a (mon.0) 380 : cluster 3 Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:15.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:14 smithi155 bash[59495]: audit 2023-10-09T06:14:14.875059+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.125:0/2311714980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:16.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:15 smithi125 bash[80002]: cluster 2023-10-09T06:14:14.572094+0000 mgr.y (mgr.25723) 256 : cluster 0 pgmap v196: 73 pgs: 2 peering, 1 active+undersized+degraded, 5 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:16.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:15 smithi125 bash[77299]: cluster 2023-10-09T06:14:14.572094+0000 mgr.y (mgr.25723) 256 : cluster 0 pgmap v196: 73 pgs: 2 peering, 1 active+undersized+degraded, 5 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:15 smithi155 bash[59495]: cluster 2023-10-09T06:14:14.572094+0000 mgr.y (mgr.25723) 256 : cluster 0 pgmap v196: 73 pgs: 2 peering, 1 active+undersized+degraded, 5 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:16 smithi125 bash[77299]: cluster 2023-10-09T06:14:16.923348+0000 mon.a (mon.0) 381 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:14:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:16 smithi125 bash[77299]: cluster 2023-10-09T06:14:16.923407+0000 mon.a (mon.0) 382 : cluster 1 Cluster is now healthy 2023-10-09T06:14:17.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:16 smithi125 bash[80002]: cluster 2023-10-09T06:14:16.923348+0000 mon.a (mon.0) 381 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:14:17.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:16 smithi125 bash[80002]: cluster 2023-10-09T06:14:16.923407+0000 mon.a (mon.0) 382 : cluster 1 Cluster is now healthy 2023-10-09T06:14:17.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:16 smithi155 bash[59495]: cluster 2023-10-09T06:14:16.923348+0000 mon.a (mon.0) 381 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-10-09T06:14:17.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:16 smithi155 bash[59495]: cluster 2023-10-09T06:14:16.923407+0000 mon.a (mon.0) 382 : cluster 1 Cluster is now healthy 2023-10-09T06:14:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:17 smithi125 bash[80002]: cluster 2023-10-09T06:14:16.572860+0000 mgr.y (mgr.25723) 257 : cluster 0 pgmap v197: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:17 smithi125 bash[80002]: audit 2023-10-09T06:14:17.234677+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.125:0/3182229553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:17 smithi125 bash[77299]: cluster 2023-10-09T06:14:16.572860+0000 mgr.y (mgr.25723) 257 : cluster 0 pgmap v197: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:17 smithi125 bash[77299]: audit 2023-10-09T06:14:17.234677+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.125:0/3182229553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:18.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:17 smithi155 bash[59495]: cluster 2023-10-09T06:14:16.572860+0000 mgr.y (mgr.25723) 257 : cluster 0 pgmap v197: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:18.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:17 smithi155 bash[59495]: audit 2023-10-09T06:14:17.234677+0000 mon.a (mon.0) 383 : audit 0 from='client.? 172.21.15.125:0/3182229553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:19.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[77299]: audit 2023-10-09T06:14:18.559413+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[77299]: audit 2023-10-09T06:14:18.561019+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[77299]: cluster 2023-10-09T06:14:18.573531+0000 mgr.y (mgr.25723) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:19.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[80002]: audit 2023-10-09T06:14:18.559413+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:19.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[80002]: audit 2023-10-09T06:14:18.561019+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:19.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:19 smithi125 bash[80002]: cluster 2023-10-09T06:14:18.573531+0000 mgr.y (mgr.25723) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:19 smithi155 bash[59495]: audit 2023-10-09T06:14:18.559413+0000 mon.a (mon.0) 384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:19 smithi155 bash[59495]: audit 2023-10-09T06:14:18.561019+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:19 smithi155 bash[59495]: cluster 2023-10-09T06:14:18.573531+0000 mgr.y (mgr.25723) 258 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:20.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:19.598534+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.125:0/3260516028' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:20.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.402743+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.411150+0000 mon.a (mon.0) 388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.412751+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:20.823 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.414150+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.421717+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.504839+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.511598+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.515110+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.517045+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[77299]: audit 2023-10-09T06:14:20.519520+0000 mon.a (mon.0) 396 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:19.598534+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.125:0/3260516028' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.402743+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.411150+0000 mon.a (mon.0) 388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.412751+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.414150+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.421717+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.504839+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.511598+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.515110+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.517045+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.825 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:20 smithi125 bash[80002]: audit 2023-10-09T06:14:20.519520+0000 mon.a (mon.0) 396 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:19.598534+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.125:0/3260516028' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.402743+0000 mon.a (mon.0) 387 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.411150+0000 mon.a (mon.0) 388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.412751+0000 mon.a (mon.0) 389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.414150+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.421717+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.504839+0000 mon.a (mon.0) 392 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.511598+0000 mon.a (mon.0) 393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.515110+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.517045+0000 mon.a (mon.0) 395 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:20.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:20 smithi155 bash[59495]: audit 2023-10-09T06:14:20.519520+0000 mon.a (mon.0) 396 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:21.578 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:21] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.33.4" 2023-10-09T06:14:21.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[77299]: audit 2023-10-09T06:14:20.519847+0000 mgr.y (mgr.25723) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:21.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[77299]: cephadm 2023-10-09T06:14:20.521293+0000 mgr.y (mgr.25723) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T06:14:21.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[77299]: cluster 2023-10-09T06:14:20.574366+0000 mgr.y (mgr.25723) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:21.866 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[80002]: audit 2023-10-09T06:14:20.519847+0000 mgr.y (mgr.25723) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:21.866 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[80002]: cephadm 2023-10-09T06:14:20.521293+0000 mgr.y (mgr.25723) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T06:14:21.866 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:21 smithi125 bash[80002]: cluster 2023-10-09T06:14:20.574366+0000 mgr.y (mgr.25723) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:21 smithi155 bash[59495]: audit 2023-10-09T06:14:20.519847+0000 mgr.y (mgr.25723) 259 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:14:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:21 smithi155 bash[59495]: cephadm 2023-10-09T06:14:20.521293+0000 mgr.y (mgr.25723) 260 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-09T06:14:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:21 smithi155 bash[59495]: cluster 2023-10-09T06:14:20.574366+0000 mgr.y (mgr.25723) 261 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:22.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:22 smithi125 bash[80002]: audit 2023-10-09T06:14:21.975795+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.125:0/1612698703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:22.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:22 smithi125 bash[77299]: audit 2023-10-09T06:14:21.975795+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.125:0/1612698703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:22.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:22 smithi155 bash[59495]: audit 2023-10-09T06:14:21.975795+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.125:0/1612698703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: cluster 2023-10-09T06:14:22.575097+0000 mgr.y (mgr.25723) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: cephadm 2023-10-09T06:14:22.890964+0000 mgr.y (mgr.25723) 263 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: audit 2023-10-09T06:14:22.900458+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: audit 2023-10-09T06:14:22.901818+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: audit 2023-10-09T06:14:22.902755+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: cephadm 2023-10-09T06:14:22.906044+0000 mgr.y (mgr.25723) 264 : cephadm 1 Deploying daemon osd.3 on smithi125 2023-10-09T06:14:24.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[80002]: audit 2023-10-09T06:14:23.678028+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: cluster 2023-10-09T06:14:22.575097+0000 mgr.y (mgr.25723) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: cephadm 2023-10-09T06:14:22.890964+0000 mgr.y (mgr.25723) 263 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: audit 2023-10-09T06:14:22.900458+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: audit 2023-10-09T06:14:22.901818+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: audit 2023-10-09T06:14:22.902755+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: cephadm 2023-10-09T06:14:22.906044+0000 mgr.y (mgr.25723) 264 : cephadm 1 Deploying daemon osd.3 on smithi125 2023-10-09T06:14:24.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:23 smithi125 bash[77299]: audit 2023-10-09T06:14:23.678028+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: cluster 2023-10-09T06:14:22.575097+0000 mgr.y (mgr.25723) 262 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: cephadm 2023-10-09T06:14:22.890964+0000 mgr.y (mgr.25723) 263 : cephadm 1 Upgrade: Updating osd.3 2023-10-09T06:14:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: audit 2023-10-09T06:14:22.900458+0000 mon.a (mon.0) 397 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: audit 2023-10-09T06:14:22.901818+0000 mon.a (mon.0) 398 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:14:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: audit 2023-10-09T06:14:22.902755+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: cephadm 2023-10-09T06:14:22.906044+0000 mgr.y (mgr.25723) 264 : cephadm 1 Deploying daemon osd.3 on smithi125 2023-10-09T06:14:24.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:23 smithi155 bash[59495]: audit 2023-10-09T06:14:23.678028+0000 mon.a (mon.0) 400 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:25.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:24 smithi125 bash[80002]: audit 2023-10-09T06:14:24.334055+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.125:0/1152227536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:25.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:24 smithi125 bash[77299]: audit 2023-10-09T06:14:24.334055+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.125:0/1152227536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:24 smithi155 bash[59495]: audit 2023-10-09T06:14:24.334055+0000 mon.a (mon.0) 401 : audit 0 from='client.? 172.21.15.125:0/1152227536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:26.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:25 smithi125 bash[77299]: cluster 2023-10-09T06:14:24.576125+0000 mgr.y (mgr.25723) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:26.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:25 smithi125 bash[80002]: cluster 2023-10-09T06:14:24.576125+0000 mgr.y (mgr.25723) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:25 smithi155 bash[59495]: cluster 2023-10-09T06:14:24.576125+0000 mgr.y (mgr.25723) 265 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:27.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:26 smithi125 bash[80002]: audit 2023-10-09T06:14:26.703066+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.125:0/2382838504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:27.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:26 smithi125 bash[77299]: audit 2023-10-09T06:14:26.703066+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.125:0/2382838504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:27.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:26 smithi155 bash[59495]: audit 2023-10-09T06:14:26.703066+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.125:0/2382838504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:27.887 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:27 smithi125 systemd[1]: Stopping Ceph osd.3 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:14:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:27 smithi125 bash[77299]: cluster 2023-10-09T06:14:26.576826+0000 mgr.y (mgr.25723) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:27 smithi125 bash[80002]: cluster 2023-10-09T06:14:26.576826+0000 mgr.y (mgr.25723) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:28.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:27 smithi125 bash[49540]: debug 2023-10-09T06:14:27.962+0000 7effc21a8700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:14:28.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:27 smithi125 bash[49540]: debug 2023-10-09T06:14:27.962+0000 7effc21a8700 -1 osd.3 99 *** Got signal Terminated *** 2023-10-09T06:14:28.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:27 smithi125 bash[49540]: debug 2023-10-09T06:14:27.962+0000 7effc21a8700 -1 osd.3 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:14:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:27 smithi155 bash[59495]: cluster 2023-10-09T06:14:26.576826+0000 mgr.y (mgr.25723) 266 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:29.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:28 smithi125 bash[77299]: cluster 2023-10-09T06:14:27.965237+0000 mon.a (mon.0) 402 : cluster 1 osd.3 marked itself down and dead 2023-10-09T06:14:29.236 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:28 smithi125 bash[80002]: cluster 2023-10-09T06:14:27.965237+0000 mon.a (mon.0) 402 : cluster 1 osd.3 marked itself down and dead 2023-10-09T06:14:29.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:28 smithi155 bash[59495]: cluster 2023-10-09T06:14:27.965237+0000 mon.a (mon.0) 402 : cluster 1 osd.3 marked itself down and dead 2023-10-09T06:14:29.929 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[93804]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T06:14:29.929 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[93898]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T06:14:30.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: cluster 2023-10-09T06:14:28.577575+0000 mgr.y (mgr.25723) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:30.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: cluster 2023-10-09T06:14:28.920752+0000 mon.a (mon.0) 403 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: cluster 2023-10-09T06:14:28.936785+0000 mon.a (mon.0) 404 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: cluster 2023-10-09T06:14:28.937118+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: cluster 2023-10-09T06:14:28.939581+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[77299]: audit 2023-10-09T06:14:29.086352+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.125:0/4172698865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: cluster 2023-10-09T06:14:28.577575+0000 mgr.y (mgr.25723) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: cluster 2023-10-09T06:14:28.920752+0000 mon.a (mon.0) 403 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: cluster 2023-10-09T06:14:28.936785+0000 mon.a (mon.0) 404 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: cluster 2023-10-09T06:14:28.937118+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: cluster 2023-10-09T06:14:28.939581+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:29 smithi125 bash[80002]: audit 2023-10-09T06:14:29.086352+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.125:0/4172698865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: cluster 2023-10-09T06:14:28.577575+0000 mgr.y (mgr.25723) 267 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: cluster 2023-10-09T06:14:28.920752+0000 mon.a (mon.0) 403 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: cluster 2023-10-09T06:14:28.936785+0000 mon.a (mon.0) 404 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: cluster 2023-10-09T06:14:28.937118+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: cluster 2023-10-09T06:14:28.939581+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e100 with expected crc 2023-10-09T06:14:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:29 smithi155 bash[59495]: audit 2023-10-09T06:14:29.086352+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.125:0/4172698865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:31.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.938477+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.939245+0000 mon.a (mon.0) 405 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T06:14:31.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.941324+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.942117+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.942270+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.943055+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[77299]: cluster 2023-10-09T06:14:29.943411+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.938477+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.939245+0000 mon.a (mon.0) 405 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.941324+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.942117+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.942270+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.943055+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:30 smithi125 bash[80002]: cluster 2023-10-09T06:14:29.943411+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.938477+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.939245+0000 mon.a (mon.0) 405 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-10-09T06:14:31.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.941324+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.942117+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.942270+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.943055+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:30 smithi155 bash[59495]: cluster 2023-10-09T06:14:29.943411+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e101 with expected crc 2023-10-09T06:14:31.469 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:31] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.33.4" 2023-10-09T06:14:32.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[77299]: cluster 2023-10-09T06:14:30.578392+0000 mgr.y (mgr.25723) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:32.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[77299]: cluster 2023-10-09T06:14:30.935046+0000 mon.a (mon.0) 406 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:32.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[77299]: audit 2023-10-09T06:14:31.442744+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.125:0/2320231652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:32.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[80002]: cluster 2023-10-09T06:14:30.578392+0000 mgr.y (mgr.25723) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:32.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[80002]: cluster 2023-10-09T06:14:30.935046+0000 mon.a (mon.0) 406 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:32.220 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:31 smithi125 bash[80002]: audit 2023-10-09T06:14:31.442744+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.125:0/2320231652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:32.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:31 smithi155 bash[59495]: cluster 2023-10-09T06:14:30.578392+0000 mgr.y (mgr.25723) 268 : cluster 0 pgmap v206: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:32.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:31 smithi155 bash[59495]: cluster 2023-10-09T06:14:30.935046+0000 mon.a (mon.0) 406 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:14:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:31 smithi155 bash[59495]: audit 2023-10-09T06:14:31.442744+0000 mon.a (mon.0) 407 : audit 0 from='client.? 172.21.15.125:0/2320231652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:32.471 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:32 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T06:14:32.471 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:32 smithi125 systemd[1]: Stopped Ceph osd.3 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:32.887 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:32 smithi125 systemd[1]: Started Ceph osd.3 for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[77299]: audit 2023-10-09T06:14:32.501983+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[77299]: audit 2023-10-09T06:14:32.510115+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[77299]: audit 2023-10-09T06:14:32.542554+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[77299]: cluster 2023-10-09T06:14:32.579151+0000 mgr.y (mgr.25723) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 2 stale+active+clean, 9 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[80002]: audit 2023-10-09T06:14:32.501983+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[80002]: audit 2023-10-09T06:14:32.510115+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[80002]: audit 2023-10-09T06:14:32.542554+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:33.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:33 smithi125 bash[80002]: cluster 2023-10-09T06:14:32.579151+0000 mgr.y (mgr.25723) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 2 stale+active+clean, 9 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:33 smithi155 bash[59495]: audit 2023-10-09T06:14:32.501983+0000 mon.a (mon.0) 408 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:33 smithi155 bash[59495]: audit 2023-10-09T06:14:32.510115+0000 mon.a (mon.0) 409 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:33 smithi155 bash[59495]: audit 2023-10-09T06:14:32.542554+0000 mon.a (mon.0) 410 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:33 smithi155 bash[59495]: cluster 2023-10-09T06:14:32.579151+0000 mgr.y (mgr.25723) 269 : cluster 0 pgmap v207: 73 pgs: 1 active+undersized+degraded, 14 peering, 2 stale+active+clean, 9 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-10-09T06:14:34.387 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T06:14:34.388 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[94081]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T06:14:34.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[77299]: cluster 2023-10-09T06:14:33.507216+0000 mon.a (mon.0) 411 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:34.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[77299]: audit 2023-10-09T06:14:33.560681+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[77299]: audit 2023-10-09T06:14:33.561712+0000 mon.a (mon.0) 413 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[77299]: audit 2023-10-09T06:14:33.801221+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.125:0/804228705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[80002]: cluster 2023-10-09T06:14:33.507216+0000 mon.a (mon.0) 411 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[80002]: audit 2023-10-09T06:14:33.560681+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[80002]: audit 2023-10-09T06:14:33.561712+0000 mon.a (mon.0) 413 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:34.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:34 smithi125 bash[80002]: audit 2023-10-09T06:14:33.801221+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.125:0/804228705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:34 smithi155 bash[59495]: cluster 2023-10-09T06:14:33.507216+0000 mon.a (mon.0) 411 : cluster 3 Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:14:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:34 smithi155 bash[59495]: audit 2023-10-09T06:14:33.560681+0000 mon.a (mon.0) 412 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:34 smithi155 bash[59495]: audit 2023-10-09T06:14:33.561712+0000 mon.a (mon.0) 413 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:34 smithi155 bash[59495]: audit 2023-10-09T06:14:33.801221+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.125:0/804228705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:35.813 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:35 smithi125 bash[77299]: cluster 2023-10-09T06:14:34.580093+0000 mgr.y (mgr.25723) 270 : cluster 0 pgmap v208: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:35.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:35 smithi125 bash[80002]: cluster 2023-10-09T06:14:34.580093+0000 mgr.y (mgr.25723) 270 : cluster 0 pgmap v208: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:35 smithi155 bash[59495]: cluster 2023-10-09T06:14:34.580093+0000 mgr.y (mgr.25723) 270 : cluster 0 pgmap v208: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:36.137 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:35 smithi125 bash[94382]: debug 2023-10-09T06:14:35.921+0000 7f73b3cab700 1 -- 172.21.15.125:0/2938440444 <== mon.0 v2:172.21.15.125:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5602db7524e0 con 0x5602db7ac800 2023-10-09T06:14:36.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:36 smithi125 bash[77299]: cluster 2023-10-09T06:14:35.503129+0000 mon.a (mon.0) 414 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:14:36.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:36 smithi125 bash[77299]: audit 2023-10-09T06:14:36.160292+0000 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.125:0/2261110710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:36.888 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:36 smithi125 bash[94382]: debug 2023-10-09T06:14:36.461+0000 7f73bce31640 -1 Falling back to public interface 2023-10-09T06:14:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:36 smithi125 bash[80002]: cluster 2023-10-09T06:14:35.503129+0000 mon.a (mon.0) 414 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:14:36.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:36 smithi125 bash[80002]: audit 2023-10-09T06:14:36.160292+0000 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.125:0/2261110710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:36 smithi155 bash[59495]: cluster 2023-10-09T06:14:35.503129+0000 mon.a (mon.0) 414 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:14:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:36 smithi155 bash[59495]: audit 2023-10-09T06:14:36.160292+0000 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.125:0/2261110710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:37 smithi125 bash[77299]: cluster 2023-10-09T06:14:36.580941+0000 mgr.y (mgr.25723) 271 : cluster 0 pgmap v209: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:37.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:37 smithi125 bash[80002]: cluster 2023-10-09T06:14:36.580941+0000 mgr.y (mgr.25723) 271 : cluster 0 pgmap v209: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:37 smithi155 bash[59495]: cluster 2023-10-09T06:14:36.580941+0000 mgr.y (mgr.25723) 271 : cluster 0 pgmap v209: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:38 smithi125 bash[77299]: cluster 2023-10-09T06:14:38.513901+0000 mon.a (mon.0) 416 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:38 smithi125 bash[80002]: cluster 2023-10-09T06:14:38.513901+0000 mon.a (mon.0) 416 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:38.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:38 smithi155 bash[59495]: cluster 2023-10-09T06:14:38.513901+0000 mon.a (mon.0) 416 : cluster 3 Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:39 smithi125 bash[80002]: audit 2023-10-09T06:14:38.538768+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.125:0/1061138068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:39 smithi125 bash[80002]: cluster 2023-10-09T06:14:38.581674+0000 mgr.y (mgr.25723) 272 : cluster 0 pgmap v210: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:39.888 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:39 smithi125 bash[94382]: debug 2023-10-09T06:14:39.689+0000 7f73bce31640 -1 osd.3 99 log_to_monitors true 2023-10-09T06:14:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:39 smithi125 bash[77299]: audit 2023-10-09T06:14:38.538768+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.125:0/1061138068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:39 smithi125 bash[77299]: cluster 2023-10-09T06:14:38.581674+0000 mgr.y (mgr.25723) 272 : cluster 0 pgmap v210: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:39 smithi155 bash[59495]: audit 2023-10-09T06:14:38.538768+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.125:0/1061138068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:39 smithi155 bash[59495]: cluster 2023-10-09T06:14:38.581674+0000 mgr.y (mgr.25723) 272 : cluster 0 pgmap v210: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:39.638271+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:39.644820+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:39.697307+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:39.697738+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:40.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:40.438643+0000 mon.a (mon.0) 420 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:40.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:40 smithi155 bash[59495]: audit 2023-10-09T06:14:40.446250+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.137 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[94382]: debug 2023-10-09T06:14:40.701+0000 7f73ac299700 -1 osd.3 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:39.638271+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:39.644820+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:39.697307+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:39.697738+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:40.438643+0000 mon.a (mon.0) 420 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[77299]: audit 2023-10-09T06:14:40.446250+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:39.638271+0000 mon.a (mon.0) 417 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:39.644820+0000 mon.a (mon.0) 418 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:39.697307+0000 mon.c (mon.2) 30 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:41.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:39.697738+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:14:41.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:40.438643+0000 mon.a (mon.0) 420 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:40 smithi125 bash[80002]: audit 2023-10-09T06:14:40.446250+0000 mon.a (mon.0) 421 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:41.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:41] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:14:42.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: cluster 2023-10-09T06:14:40.582686+0000 mgr.y (mgr.25723) 273 : cluster 0 pgmap v211: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: audit 2023-10-09T06:14:40.652312+0000 mon.a (mon.0) 422 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: cluster 2023-10-09T06:14:40.652413+0000 mon.a (mon.0) 423 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: audit 2023-10-09T06:14:40.653308+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: audit 2023-10-09T06:14:40.653775+0000 mon.a (mon.0) 424 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[77299]: audit 2023-10-09T06:14:40.902925+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.125:0/224739250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: cluster 2023-10-09T06:14:40.582686+0000 mgr.y (mgr.25723) 273 : cluster 0 pgmap v211: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:42.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: audit 2023-10-09T06:14:40.652312+0000 mon.a (mon.0) 422 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:14:42.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: cluster 2023-10-09T06:14:40.652413+0000 mon.a (mon.0) 423 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T06:14:42.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: audit 2023-10-09T06:14:40.653308+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: audit 2023-10-09T06:14:40.653775+0000 mon.a (mon.0) 424 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:41 smithi125 bash[80002]: audit 2023-10-09T06:14:40.902925+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.125:0/224739250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: cluster 2023-10-09T06:14:40.582686+0000 mgr.y (mgr.25723) 273 : cluster 0 pgmap v211: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: audit 2023-10-09T06:14:40.652312+0000 mon.a (mon.0) 422 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: cluster 2023-10-09T06:14:40.652413+0000 mon.a (mon.0) 423 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: audit 2023-10-09T06:14:40.653308+0000 mon.c (mon.2) 31 : audit 1 from='osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: audit 2023-10-09T06:14:40.653775+0000 mon.a (mon.0) 424 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi125", "root=default"]}]: dispatch 2023-10-09T06:14:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:41 smithi155 bash[59495]: audit 2023-10-09T06:14:40.902925+0000 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.125:0/224739250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:43.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: cluster 2023-10-09T06:14:41.647088+0000 mon.a (mon.0) 426 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: cluster 2023-10-09T06:14:41.653236+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: cluster 2023-10-09T06:14:41.654050+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: cluster 2023-10-09T06:14:41.654146+0000 mon.a (mon.0) 427 : cluster 1 osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101] boot 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: cluster 2023-10-09T06:14:41.654234+0000 mon.a (mon.0) 428 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[80002]: audit 2023-10-09T06:14:41.655340+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: cluster 2023-10-09T06:14:41.647088+0000 mon.a (mon.0) 426 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: cluster 2023-10-09T06:14:41.653236+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: cluster 2023-10-09T06:14:41.654050+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: cluster 2023-10-09T06:14:41.654146+0000 mon.a (mon.0) 427 : cluster 1 osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101] boot 2023-10-09T06:14:43.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: cluster 2023-10-09T06:14:41.654234+0000 mon.a (mon.0) 428 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:14:43.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:42 smithi125 bash[77299]: audit 2023-10-09T06:14:41.655340+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: cluster 2023-10-09T06:14:41.647088+0000 mon.a (mon.0) 426 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: cluster 2023-10-09T06:14:41.653236+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: cluster 2023-10-09T06:14:41.654050+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: cluster 2023-10-09T06:14:41.654146+0000 mon.a (mon.0) 427 : cluster 1 osd.3 [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101] boot 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: cluster 2023-10-09T06:14:41.654234+0000 mon.a (mon.0) 428 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:14:43.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:42 smithi155 bash[59495]: audit 2023-10-09T06:14:41.655340+0000 mon.a (mon.0) 429 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:14:43.290 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T06:14:43.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.583541+0000 mgr.y (mgr.25723) 274 : cluster 0 pgmap v214: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.668622+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.669795+0000 mon.a (mon.0) 430 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.670395+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.672878+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.674391+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.676008+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.677044+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: cluster 2023-10-09T06:14:42.678805+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[80002]: audit 2023-10-09T06:14:43.267049+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.125:0/2867327841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.583541+0000 mgr.y (mgr.25723) 274 : cluster 0 pgmap v214: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.668622+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.669795+0000 mon.a (mon.0) 430 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.670395+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.672878+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.674391+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.676008+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.677044+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: cluster 2023-10-09T06:14:42.678805+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:43.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:43 smithi125 bash[77299]: audit 2023-10-09T06:14:43.267049+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.125:0/2867327841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.583541+0000 mgr.y (mgr.25723) 274 : cluster 0 pgmap v214: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-09T06:14:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.668622+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.669795+0000 mon.a (mon.0) 430 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:14:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.670395+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.672878+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.674391+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.676008+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.677044+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: cluster 2023-10-09T06:14:42.678805+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e104 with expected crc 2023-10-09T06:14:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:43 smithi155 bash[59495]: audit 2023-10-09T06:14:43.267049+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.125:0/2867327841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:44.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:14:45.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:44 smithi125 bash[80002]: audit 2023-10-09T06:14:43.696759+0000 mon.a (mon.0) 431 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:45.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:44 smithi125 bash[77299]: audit 2023-10-09T06:14:43.696759+0000 mon.a (mon.0) 431 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:45.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:44 smithi155 bash[59495]: audit 2023-10-09T06:14:43.696759+0000 mon.a (mon.0) 431 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:45 smithi125 bash[80002]: cluster 2023-10-09T06:14:44.584522+0000 mgr.y (mgr.25723) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 3 active+undersized+degraded, 15 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:14:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:45 smithi125 bash[80002]: cluster 2023-10-09T06:14:44.692710+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:46.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:45 smithi125 bash[77299]: cluster 2023-10-09T06:14:44.584522+0000 mgr.y (mgr.25723) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 3 active+undersized+degraded, 15 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:14:46.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:45 smithi125 bash[77299]: cluster 2023-10-09T06:14:44.692710+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:45 smithi155 bash[59495]: cluster 2023-10-09T06:14:44.584522+0000 mgr.y (mgr.25723) 275 : cluster 0 pgmap v216: 73 pgs: 5 peering, 3 active+undersized+degraded, 15 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-09T06:14:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:45 smithi155 bash[59495]: cluster 2023-10-09T06:14:44.692710+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:47.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:46 smithi125 bash[77299]: cluster 2023-10-09T06:14:46.704360+0000 mon.a (mon.0) 433 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-10-09T06:14:47.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:46 smithi125 bash[77299]: cluster 2023-10-09T06:14:46.704389+0000 mon.a (mon.0) 434 : cluster 1 Cluster is now healthy 2023-10-09T06:14:47.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:46 smithi125 bash[80002]: cluster 2023-10-09T06:14:46.704360+0000 mon.a (mon.0) 433 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-10-09T06:14:47.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:46 smithi125 bash[80002]: cluster 2023-10-09T06:14:46.704389+0000 mon.a (mon.0) 434 : cluster 1 Cluster is now healthy 2023-10-09T06:14:47.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:46 smithi155 bash[59495]: cluster 2023-10-09T06:14:46.704360+0000 mon.a (mon.0) 433 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-10-09T06:14:47.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:46 smithi155 bash[59495]: cluster 2023-10-09T06:14:46.704389+0000 mon.a (mon.0) 434 : cluster 1 Cluster is now healthy 2023-10-09T06:14:48.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:47 smithi125 bash[77299]: cluster 2023-10-09T06:14:46.585157+0000 mgr.y (mgr.25723) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:48.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:47 smithi125 bash[80002]: cluster 2023-10-09T06:14:46.585157+0000 mgr.y (mgr.25723) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:47 smithi155 bash[59495]: cluster 2023-10-09T06:14:46.585157+0000 mgr.y (mgr.25723) 276 : cluster 0 pgmap v217: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:49.545 INFO:teuthology.orchestra.run.smithi125.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:14:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[77299]: audit 2023-10-09T06:14:48.562374+0000 mon.a (mon.0) 435 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[77299]: audit 2023-10-09T06:14:48.564140+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[77299]: cluster 2023-10-09T06:14:48.585515+0000 mgr.y (mgr.25723) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[77299]: audit 2023-10-09T06:14:49.541229+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[80002]: audit 2023-10-09T06:14:48.562374+0000 mon.a (mon.0) 435 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[80002]: audit 2023-10-09T06:14:48.564140+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[80002]: cluster 2023-10-09T06:14:48.585515+0000 mgr.y (mgr.25723) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:49 smithi125 bash[80002]: audit 2023-10-09T06:14:49.541229+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:49 smithi155 bash[59495]: audit 2023-10-09T06:14:48.562374+0000 mon.a (mon.0) 435 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:49 smithi155 bash[59495]: audit 2023-10-09T06:14:48.564140+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:49 smithi155 bash[59495]: cluster 2023-10-09T06:14:48.585515+0000 mgr.y (mgr.25723) 277 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:49 smithi155 bash[59495]: audit 2023-10-09T06:14:49.541229+0000 mon.a (mon.0) 437 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:50.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:14:50.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:50 smithi125 bash[77299]: audit 2023-10-09T06:14:49.535343+0000 mgr.y (mgr.25723) 278 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:50.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:50 smithi125 bash[77299]: cephadm 2023-10-09T06:14:49.535733+0000 mgr.y (mgr.25723) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:14:50.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:50 smithi125 bash[80002]: audit 2023-10-09T06:14:49.535343+0000 mgr.y (mgr.25723) 278 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:50.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:50 smithi125 bash[80002]: cephadm 2023-10-09T06:14:49.535733+0000 mgr.y (mgr.25723) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:14:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:50 smithi155 bash[59495]: audit 2023-10-09T06:14:49.535343+0000 mgr.y (mgr.25723) 278 : audit 0 from='client.34557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:50 smithi155 bash[59495]: cephadm 2023-10-09T06:14:49.535733+0000 mgr.y (mgr.25723) 279 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:14:51.578 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:14:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:14:51] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T06:14:51.579 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:51 smithi125 bash[80002]: cluster 2023-10-09T06:14:50.586366+0000 mgr.y (mgr.25723) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:51 smithi125 bash[77299]: cluster 2023-10-09T06:14:50.586366+0000 mgr.y (mgr.25723) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:51.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:51 smithi155 bash[59495]: cluster 2023-10-09T06:14:50.586366+0000 mgr.y (mgr.25723) 280 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: cluster 2023-10-09T06:14:52.587191+0000 mgr.y (mgr.25723) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.803125+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.811338+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.812810+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.814302+0000 mon.a (mon.0) 441 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.822278+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.902438+0000 mon.a (mon.0) 443 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.904598+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.905614+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:52.914335+0000 mon.a (mon.0) 446 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[77299]: audit 2023-10-09T06:14:53.713815+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: cluster 2023-10-09T06:14:52.587191+0000 mgr.y (mgr.25723) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.803125+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.811338+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.812810+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.814302+0000 mon.a (mon.0) 441 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.822278+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.902438+0000 mon.a (mon.0) 443 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:54.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.904598+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.905614+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:52.914335+0000 mon.a (mon.0) 446 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:53 smithi125 bash[80002]: audit 2023-10-09T06:14:53.713815+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: cluster 2023-10-09T06:14:52.587191+0000 mgr.y (mgr.25723) 281 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.803125+0000 mon.a (mon.0) 438 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.811338+0000 mon.a (mon.0) 439 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.812810+0000 mon.a (mon.0) 440 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.814302+0000 mon.a (mon.0) 441 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.822278+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.902438+0000 mon.a (mon.0) 443 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.904598+0000 mon.a (mon.0) 444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.905614+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:52.914335+0000 mon.a (mon.0) 446 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:53 smithi155 bash[59495]: audit 2023-10-09T06:14:53.713815+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:14:54.969 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:14:54.969 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (8m) - 14m 13.7M - 0.23.0 ba2b418f427c 287b260dd8a4 2023-10-09T06:14:54.970 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (8m) - 14m 41.5M - 8.3.5 dad864ee21e9 988e098b0939 2023-10-09T06:14:54.970 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (13m) - 13m 17.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:14:54.970 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (12m) - 12m 14.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:14:54.970 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (7m) - 21m 429M - 18.2.0-598-gb4fa663f 3060897fdf97 6a90d5aaacfd 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (7m) - 23m 483M - 18.2.0-598-gb4fa663f 3060897fdf97 4731d4d85533 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (6m) - 23m 72.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 66d70cab6c5d 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (3m) - 22m 41.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 1d811296fe1c 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (5m) - 22m 47.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 bedca0a9cc9d 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (8m) - 15m 9308k - 1.3.1 1dbe0e931976 951da6650ff3 2023-10-09T06:14:54.971 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (8m) - 15m 9568k - 1.3.1 1dbe0e931976 4e060b6a5bef 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (108s) - 21m 43.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 144870f72a77 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (79s) - 20m 42.5M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 5f942fed44c4 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (49s) - 19m 40.9M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 6ef67efb5f79 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (20s) - 19m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 09a8888aee8b 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (18m) - 18m 46.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (17m) - 17m 46.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (16m) - 16m 46.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (16m) - 16m 51.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:14:54.972 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (7m) - 15m 58.2M - 2.33.4 514e6a882f6e 5a577340c8e2 2023-10-09T06:14:55.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:14:56.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:55 smithi125 bash[80002]: cluster 2023-10-09T06:14:54.588197+0000 mgr.y (mgr.25723) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:55 smithi125 bash[80002]: audit 2023-10-09T06:14:54.952271+0000 mgr.y (mgr.25723) 283 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:55 smithi125 bash[77299]: cluster 2023-10-09T06:14:54.588197+0000 mgr.y (mgr.25723) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:55 smithi125 bash[77299]: audit 2023-10-09T06:14:54.952271+0000 mgr.y (mgr.25723) 283 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:55 smithi155 bash[59495]: cluster 2023-10-09T06:14:54.588197+0000 mgr.y (mgr.25723) 282 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:55 smithi155 bash[59495]: audit 2023-10-09T06:14:54.952271+0000 mgr.y (mgr.25723) 283 : audit 0 from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:14:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:57 smithi125 bash[77299]: cluster 2023-10-09T06:14:56.589007+0000 mgr.y (mgr.25723) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:57 smithi125 bash[80002]: cluster 2023-10-09T06:14:56.589007+0000 mgr.y (mgr.25723) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:58.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:57 smithi155 bash[59495]: cluster 2023-10-09T06:14:56.589007+0000 mgr.y (mgr.25723) 284 : cluster 0 pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:59.293 INFO:teuthology.orchestra.run.smithi125.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:14:59.293 INFO:teuthology.orchestra.run.smithi125.stdout:alertmanager.a smithi125 *:9093,9094 running (8m) - 14m 13.7M - 0.23.0 ba2b418f427c 287b260dd8a4 2023-10-09T06:14:59.293 INFO:teuthology.orchestra.run.smithi125.stdout:grafana.a smithi155 *:3000 running (8m) - 14m 41.5M - 8.3.5 dad864ee21e9 988e098b0939 2023-10-09T06:14:59.293 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi125.yetyhy smithi125 running (13m) - 13m 17.2M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 43f152e3b628 2023-10-09T06:14:59.293 INFO:teuthology.orchestra.run.smithi125.stdout:mds.foo.smithi155.itxbmb smithi155 running (13m) - 13m 14.4M - 16.2.14-48-gaeb2e425 83c1f8bb3ad8 65fd3b2f7f9c 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.x smithi155 *:8443,9283,8765 running (7m) - 21m 429M - 18.2.0-598-gb4fa663f 3060897fdf97 6a90d5aaacfd 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:mgr.y smithi125 *:8443,9283,8765 running (8m) - 23m 483M - 18.2.0-598-gb4fa663f 3060897fdf97 4731d4d85533 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a smithi125 running (6m) - 24m 72.9M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 66d70cab6c5d 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b smithi155 running (3m) - 22m 41.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 1d811296fe1c 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c smithi125 running (5m) - 22m 47.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 bedca0a9cc9d 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.a smithi125 *:9100 running (8m) - 15m 9308k - 1.3.1 1dbe0e931976 951da6650ff3 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:node-exporter.b smithi155 *:9100 running (8m) - 15m 9568k - 1.3.1 1dbe0e931976 4e060b6a5bef 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 smithi125 running (112s) - 21m 43.4M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 144870f72a77 2023-10-09T06:14:59.294 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 smithi125 running (83s) - 20m 42.5M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 5f942fed44c4 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 smithi125 running (54s) - 19m 40.9M 2524M 18.2.0-598-gb4fa663f 3060897fdf97 6ef67efb5f79 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 smithi125 running (24s) - 19m - 2524M 18.2.0-598-gb4fa663f 3060897fdf97 09a8888aee8b 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 smithi155 running (18m) - 18m 46.2M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 b01bf390ef27 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 smithi155 running (17m) - 17m 46.0M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 4db7e684c5be 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 smithi155 running (16m) - 17m 46.1M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 f652fb070860 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 smithi155 running (16m) - 16m 51.3M 2524M 16.2.14-48-gaeb2e425 83c1f8bb3ad8 e84683d25563 2023-10-09T06:14:59.295 INFO:teuthology.orchestra.run.smithi125.stdout:prometheus.a smithi155 *:9095 running (7m) - 15m 58.2M - 2.33.4 514e6a882f6e 5a577340c8e2 2023-10-09T06:14:59.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:14:59 smithi125 bash[80002]: cluster 2023-10-09T06:14:58.589768+0000 mgr.y (mgr.25723) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:59.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:14:59 smithi125 bash[77299]: cluster 2023-10-09T06:14:58.589768+0000 mgr.y (mgr.25723) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:14:59.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:14:59 smithi155 bash[59495]: cluster 2023-10-09T06:14:58.589768+0000 mgr.y (mgr.25723) 285 : cluster 0 pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:00.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T06:15:00.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:00 smithi125 bash[77299]: audit 2023-10-09T06:14:59.271376+0000 mgr.y (mgr.25723) 286 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:15:00.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:00 smithi125 bash[80002]: audit 2023-10-09T06:14:59.271376+0000 mgr.y (mgr.25723) 286 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:15:00.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:00 smithi155 bash[59495]: audit 2023-10-09T06:14:59.271376+0000 mgr.y (mgr.25723) 286 : audit 0 from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:15:01.534 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:01] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T06:15:01.837 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:01 smithi155 bash[59495]: cluster 2023-10-09T06:15:00.590529+0000 mgr.y (mgr.25723) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:01 smithi125 bash[77299]: cluster 2023-10-09T06:15:00.590529+0000 mgr.y (mgr.25723) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:01 smithi125 bash[80002]: cluster 2023-10-09T06:15:00.590529+0000 mgr.y (mgr.25723) 287 : cluster 0 pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:03.735 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: "mon": { 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr": { 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T06:15:03.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "osd": { 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 4, 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "mds": { 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 2 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:15:03.737 INFO:teuthology.orchestra.run.smithi125.stdout: "overall": { 2023-10-09T06:15:03.738 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 16.2.14-48-gaeb2e425 (aeb2e425bccd1202ba613f25c784b52a47922a5f) pacific (stable)": 6, 2023-10-09T06:15:03.738 INFO:teuthology.orchestra.run.smithi125.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T06:15:03.738 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:15:03.738 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:15:04.030 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:03 smithi125 bash[77299]: cluster 2023-10-09T06:15:02.591312+0000 mgr.y (mgr.25723) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:04.030 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:03 smithi125 bash[77299]: audit 2023-10-09T06:15:03.556908+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:04.030 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:03 smithi125 bash[80002]: cluster 2023-10-09T06:15:02.591312+0000 mgr.y (mgr.25723) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:04.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:03 smithi125 bash[80002]: audit 2023-10-09T06:15:03.556908+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:03 smithi155 bash[59495]: cluster 2023-10-09T06:15:02.591312+0000 mgr.y (mgr.25723) 288 : cluster 0 pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:03 smithi155 bash[59495]: audit 2023-10-09T06:15:03.556908+0000 mon.a (mon.0) 448 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:04.645 DEBUG:teuthology.parallel:result is None 2023-10-09T06:15:04.646 INFO:teuthology.run_tasks:Running task print... 2023-10-09T06:15:04.655 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T06:15:04.655 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T06:15:04.664 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T06:15:04.664 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T06:15:04.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T06:15:05.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:04 smithi125 bash[77299]: audit 2023-10-09T06:15:03.731318+0000 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.125:0/305873893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:05.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:04 smithi125 bash[80002]: audit 2023-10-09T06:15:03.731318+0000 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.125:0/305873893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:04 smithi155 bash[59495]: audit 2023-10-09T06:15:03.731318+0000 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.125:0/305873893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:06.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:05 smithi125 bash[77299]: cluster 2023-10-09T06:15:04.591966+0000 mgr.y (mgr.25723) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:06.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:05 smithi125 bash[80002]: cluster 2023-10-09T06:15:04.591966+0000 mgr.y (mgr.25723) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:05 smithi155 bash[59495]: cluster 2023-10-09T06:15:04.591966+0000 mgr.y (mgr.25723) 289 : cluster 0 pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:07 smithi125 bash[77299]: cluster 2023-10-09T06:15:06.592702+0000 mgr.y (mgr.25723) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:08.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:07 smithi125 bash[80002]: cluster 2023-10-09T06:15:06.592702+0000 mgr.y (mgr.25723) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:08.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:07 smithi155 bash[59495]: cluster 2023-10-09T06:15:06.592702+0000 mgr.y (mgr.25723) 290 : cluster 0 pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:09.439 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:09 smithi125 bash[77299]: cluster 2023-10-09T06:15:08.593419+0000 mgr.y (mgr.25723) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:09 smithi125 bash[80002]: cluster 2023-10-09T06:15:08.593419+0000 mgr.y (mgr.25723) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:09 smithi155 bash[59495]: cluster 2023-10-09T06:15:08.593419+0000 mgr.y (mgr.25723) 291 : cluster 0 pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:11.526 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:11] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:15:11.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:11 smithi125 bash[80002]: cluster 2023-10-09T06:15:10.594312+0000 mgr.y (mgr.25723) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:11.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:11 smithi125 bash[77299]: cluster 2023-10-09T06:15:10.594312+0000 mgr.y (mgr.25723) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:11 smithi155 bash[59495]: cluster 2023-10-09T06:15:10.594312+0000 mgr.y (mgr.25723) 292 : cluster 0 pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:13.075 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:15:13.867 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":36,"active_gid":25723,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6800","nonce":1967904048},{"type":"v1","addr":"172.21.15.125:6801","nonce":1967904048}]},"active_addr":"172.21.15.125:6801/1967904048","active_change":"2023-10-09T06:08:18.401865+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25717,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":84,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":298048675}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3303697467}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":3632902431}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.125:0","nonce":913248877}]}]} 2023-10-09T06:15:13.872 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T06:15:13.872 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:15:13.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:15:14.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:13 smithi125 bash[77299]: cluster 2023-10-09T06:15:12.595188+0000 mgr.y (mgr.25723) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:13 smithi125 bash[77299]: audit 2023-10-09T06:15:13.062616+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.125:0/1992188462' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:15:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:13 smithi125 bash[80002]: cluster 2023-10-09T06:15:12.595188+0000 mgr.y (mgr.25723) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:13 smithi125 bash[80002]: audit 2023-10-09T06:15:13.062616+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.125:0/1992188462' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:15:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:13 smithi155 bash[59495]: cluster 2023-10-09T06:15:12.595188+0000 mgr.y (mgr.25723) 293 : cluster 0 pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:13 smithi155 bash[59495]: audit 2023-10-09T06:15:13.062616+0000 mon.a (mon.0) 450 : audit 0 from='client.? 172.21.15.125:0/1992188462' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:15:16.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:15 smithi125 bash[77299]: cluster 2023-10-09T06:15:14.596073+0000 mgr.y (mgr.25723) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:15 smithi125 bash[80002]: cluster 2023-10-09T06:15:14.596073+0000 mgr.y (mgr.25723) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:15 smithi155 bash[59495]: cluster 2023-10-09T06:15:14.596073+0000 mgr.y (mgr.25723) 294 : cluster 0 pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:17 smithi125 bash[77299]: cluster 2023-10-09T06:15:16.596877+0000 mgr.y (mgr.25723) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:17 smithi125 bash[80002]: cluster 2023-10-09T06:15:16.596877+0000 mgr.y (mgr.25723) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:18.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:17 smithi155 bash[59495]: cluster 2023-10-09T06:15:16.596877+0000 mgr.y (mgr.25723) 295 : cluster 0 pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:18.653 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:19 smithi125 bash[77299]: audit 2023-10-09T06:15:18.557907+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:19.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:19 smithi125 bash[77299]: cluster 2023-10-09T06:15:18.597668+0000 mgr.y (mgr.25723) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:19 smithi125 bash[80002]: audit 2023-10-09T06:15:18.557907+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:19 smithi125 bash[80002]: cluster 2023-10-09T06:15:18.597668+0000 mgr.y (mgr.25723) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:19 smithi155 bash[59495]: audit 2023-10-09T06:15:18.557907+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:19 smithi155 bash[59495]: cluster 2023-10-09T06:15:18.597668+0000 mgr.y (mgr.25723) 296 : cluster 0 pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:21.487 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:21] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T06:15:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:21 smithi125 bash[77299]: cluster 2023-10-09T06:15:20.598528+0000 mgr.y (mgr.25723) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:21.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:21 smithi125 bash[80002]: cluster 2023-10-09T06:15:20.598528+0000 mgr.y (mgr.25723) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:21.919 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:15:21.920 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":104,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:14:42.656910+0000","last_up_change":"2023-10-09T06:14:41.646246+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6803","nonce":155836221}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6805","nonce":155836221}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6809","nonce":155836221}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6807","nonce":155836221}]},"public_addr":"172.21.15.125:6803/155836221","cluster_addr":"172.21.15.125:6805/155836221","heartbeat_back_addr":"172.21.15.125:6809/155836221","heartbeat_front_addr":"172.21.15.125:6807/155836221","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6811","nonce":3444691262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6813","nonce":3444691262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6817","nonce":3444691262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6815","nonce":3444691262}]},"public_addr":"172.21.15.125:6811/3444691262","cluster_addr":"172.21.15.125:6813/3444691262","heartbeat_back_addr":"172.21.15.125:6817/3444691262","heartbeat_front_addr":"172.21.15.125:6815/3444691262","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6819","nonce":1172219518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6821","nonce":1172219518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6825","nonce":1172219518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6823","nonce":1172219518}]},"public_addr":"172.21.15.125:6819/1172219518","cluster_addr":"172.21.15.125:6821/1172219518","heartbeat_back_addr":"172.21.15.125:6825/1172219518","heartbeat_front_addr":"172.21.15.125:6823/1172219518","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6827","nonce":2373950101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6829","nonce":2373950101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6833","nonce":2373950101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6831","nonce":2373950101}]},"public_addr":"172.21.15.125:6827/2373950101","cluster_addr":"172.21.15.125:6829/2373950101","heartbeat_back_addr":"172.21.15.125:6833/2373950101","heartbeat_front_addr":"172.21.15.125:6831/2373950101","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/2757914842":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/1505504676":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/4007395716":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/2669712956":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/4000145196":"2023-10-10T06:08:18.401578+0000","172.21.15.155:6834/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/928092871":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/457536542":"2023-10-10T06:05:34.904573+0000","172.21.15.125:6800/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.125:6801/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/3175453899":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/4031419935":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/497767166":"2023-10-10T06:07:14.674105+0000","172.21.15.155:6835/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/4016621680":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/3959201196":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:15:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:21 smithi155 bash[59495]: cluster 2023-10-09T06:15:20.598528+0000 mgr.y (mgr.25723) 297 : cluster 0 pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:22.812 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:15:22.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:15:23.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:22 smithi125 bash[77299]: audit 2023-10-09T06:15:21.913023+0000 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.125:0/1816644397' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:23.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:22 smithi125 bash[80002]: audit 2023-10-09T06:15:21.913023+0000 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.125:0/1816644397' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:23.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:22 smithi155 bash[59495]: audit 2023-10-09T06:15:21.913023+0000 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.125:0/1816644397' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:24.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:23 smithi125 bash[77299]: cluster 2023-10-09T06:15:22.599275+0000 mgr.y (mgr.25723) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:24.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:23 smithi125 bash[80002]: cluster 2023-10-09T06:15:22.599275+0000 mgr.y (mgr.25723) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:23 smithi155 bash[59495]: cluster 2023-10-09T06:15:22.599275+0000 mgr.y (mgr.25723) 298 : cluster 0 pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:26.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:25 smithi125 bash[77299]: cluster 2023-10-09T06:15:24.600150+0000 mgr.y (mgr.25723) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:26.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:25 smithi125 bash[80002]: cluster 2023-10-09T06:15:24.600150+0000 mgr.y (mgr.25723) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:26.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:25 smithi155 bash[59495]: cluster 2023-10-09T06:15:24.600150+0000 mgr.y (mgr.25723) 299 : cluster 0 pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:27.599 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:27.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:27 smithi125 bash[77299]: cluster 2023-10-09T06:15:26.600956+0000 mgr.y (mgr.25723) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:27.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:27 smithi125 bash[80002]: cluster 2023-10-09T06:15:26.600956+0000 mgr.y (mgr.25723) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:28.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:27 smithi155 bash[59495]: cluster 2023-10-09T06:15:26.600956+0000 mgr.y (mgr.25723) 300 : cluster 0 pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:29.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:29 smithi125 bash[77299]: cluster 2023-10-09T06:15:28.601448+0000 mgr.y (mgr.25723) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:29 smithi125 bash[80002]: cluster 2023-10-09T06:15:28.601448+0000 mgr.y (mgr.25723) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:29.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:29 smithi155 bash[59495]: cluster 2023-10-09T06:15:28.601448+0000 mgr.y (mgr.25723) 301 : cluster 0 pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:30.990 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:15:30.990 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":104,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:14:42.656910+0000","last_up_change":"2023-10-09T06:14:41.646246+0000","last_in_change":"2023-10-09T05:58:24.566938+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":103,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6803","nonce":155836221}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6805","nonce":155836221}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6809","nonce":155836221}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6807","nonce":155836221}]},"public_addr":"172.21.15.125:6803/155836221","cluster_addr":"172.21.15.125:6805/155836221","heartbeat_back_addr":"172.21.15.125:6809/155836221","heartbeat_front_addr":"172.21.15.125:6807/155836221","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":103,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6811","nonce":3444691262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6813","nonce":3444691262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6817","nonce":3444691262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6815","nonce":3444691262}]},"public_addr":"172.21.15.125:6811/3444691262","cluster_addr":"172.21.15.125:6813/3444691262","heartbeat_back_addr":"172.21.15.125:6817/3444691262","heartbeat_front_addr":"172.21.15.125:6815/3444691262","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":103,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6819","nonce":1172219518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6821","nonce":1172219518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6825","nonce":1172219518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6823","nonce":1172219518}]},"public_addr":"172.21.15.125:6819/1172219518","cluster_addr":"172.21.15.125:6821/1172219518","heartbeat_back_addr":"172.21.15.125:6825/1172219518","heartbeat_front_addr":"172.21.15.125:6823/1172219518","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":103,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6827","nonce":2373950101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6829","nonce":2373950101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6833","nonce":2373950101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6831","nonce":2373950101}]},"public_addr":"172.21.15.125:6827/2373950101","cluster_addr":"172.21.15.125:6829/2373950101","heartbeat_back_addr":"172.21.15.125:6833/2373950101","heartbeat_front_addr":"172.21.15.125:6831/2373950101","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":100,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":103,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.155:6834/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/2757914842":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1505504676":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.155:0/3959201196":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/928092871":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/457536542":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/4007395716":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/2669712956":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6800/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/4000145196":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/4016621680":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.155:0/497767166":"2023-10-10T06:07:14.674105+0000","172.21.15.155:6835/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/3175453899":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/4031419935":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6801/105318502":"2023-10-10T06:08:18.401578+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:15:31.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:31] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T06:15:31.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:15:31.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:15:31.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:15:31.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:15:31.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:15:31.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:15:31.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:15:31.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:15:32.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:31 smithi125 bash[77299]: cluster 2023-10-09T06:15:30.602074+0000 mgr.y (mgr.25723) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:32.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:31 smithi125 bash[77299]: audit 2023-10-09T06:15:30.982700+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.125:0/811120124' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:32.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:31 smithi125 bash[80002]: cluster 2023-10-09T06:15:30.602074+0000 mgr.y (mgr.25723) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:32.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:31 smithi125 bash[80002]: audit 2023-10-09T06:15:30.982700+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.125:0/811120124' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:32.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:31 smithi155 bash[59495]: cluster 2023-10-09T06:15:30.602074+0000 mgr.y (mgr.25723) 302 : cluster 0 pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:31 smithi155 bash[59495]: audit 2023-10-09T06:15:30.982700+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.125:0/811120124' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:15:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:33 smithi125 bash[77299]: cluster 2023-10-09T06:15:32.602984+0000 mgr.y (mgr.25723) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:33 smithi125 bash[77299]: audit 2023-10-09T06:15:33.558136+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:33 smithi125 bash[80002]: cluster 2023-10-09T06:15:32.602984+0000 mgr.y (mgr.25723) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:33 smithi125 bash[80002]: audit 2023-10-09T06:15:33.558136+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:33 smithi155 bash[59495]: cluster 2023-10-09T06:15:32.602984+0000 mgr.y (mgr.25723) 303 : cluster 0 pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:33 smithi155 bash[59495]: audit 2023-10-09T06:15:33.558136+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:36.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:35 smithi125 bash[77299]: cluster 2023-10-09T06:15:34.603956+0000 mgr.y (mgr.25723) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:36.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:35 smithi125 bash[80002]: cluster 2023-10-09T06:15:34.603956+0000 mgr.y (mgr.25723) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:35 smithi155 bash[59495]: cluster 2023-10-09T06:15:34.603956+0000 mgr.y (mgr.25723) 304 : cluster 0 pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:36.710 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.711 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.712 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.712 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.712 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.713 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.713 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:36.721 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:38.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:37 smithi125 bash[77299]: cluster 2023-10-09T06:15:36.604741+0000 mgr.y (mgr.25723) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:38.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:37 smithi125 bash[80002]: cluster 2023-10-09T06:15:36.604741+0000 mgr.y (mgr.25723) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:38.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:37 smithi155 bash[59495]: cluster 2023-10-09T06:15:36.604741+0000 mgr.y (mgr.25723) 305 : cluster 0 pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:39.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:39 smithi125 bash[77299]: cluster 2023-10-09T06:15:38.605471+0000 mgr.y (mgr.25723) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:39.633 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:39 smithi125 bash[80002]: cluster 2023-10-09T06:15:38.605471+0000 mgr.y (mgr.25723) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:39 smithi155 bash[59495]: cluster 2023-10-09T06:15:38.605471+0000 mgr.y (mgr.25723) 306 : cluster 0 pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:41.525 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:41] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.33.4" 2023-10-09T06:15:41.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:41 smithi125 bash[77299]: cluster 2023-10-09T06:15:40.606027+0000 mgr.y (mgr.25723) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:41.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:41 smithi125 bash[80002]: cluster 2023-10-09T06:15:40.606027+0000 mgr.y (mgr.25723) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:41 smithi155 bash[59495]: cluster 2023-10-09T06:15:40.606027+0000 mgr.y (mgr.25723) 307 : cluster 0 pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:43.951 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:43 smithi125 bash[77299]: cluster 2023-10-09T06:15:42.606638+0000 mgr.y (mgr.25723) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:43.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:43 smithi125 bash[80002]: cluster 2023-10-09T06:15:42.606638+0000 mgr.y (mgr.25723) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:43 smithi155 bash[59495]: cluster 2023-10-09T06:15:42.606638+0000 mgr.y (mgr.25723) 308 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:45 smithi125 bash[80002]: cluster 2023-10-09T06:15:44.607189+0000 mgr.y (mgr.25723) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:45.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:45 smithi125 bash[77299]: cluster 2023-10-09T06:15:44.607189+0000 mgr.y (mgr.25723) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:45 smithi155 bash[59495]: cluster 2023-10-09T06:15:44.607189+0000 mgr.y (mgr.25723) 309 : cluster 0 pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:47.938 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:47 smithi125 bash[77299]: cluster 2023-10-09T06:15:46.608042+0000 mgr.y (mgr.25723) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:47.939 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:47 smithi125 bash[80002]: cluster 2023-10-09T06:15:46.608042+0000 mgr.y (mgr.25723) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:47 smithi155 bash[59495]: cluster 2023-10-09T06:15:46.608042+0000 mgr.y (mgr.25723) 310 : cluster 0 pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:49 smithi125 bash[77299]: audit 2023-10-09T06:15:48.558714+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:49.914 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:49 smithi125 bash[77299]: cluster 2023-10-09T06:15:48.609084+0000 mgr.y (mgr.25723) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:49.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:49 smithi125 bash[80002]: audit 2023-10-09T06:15:48.558714+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:49.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:49 smithi125 bash[80002]: cluster 2023-10-09T06:15:48.609084+0000 mgr.y (mgr.25723) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:49.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:49 smithi155 bash[59495]: audit 2023-10-09T06:15:48.558714+0000 mon.a (mon.0) 454 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:49.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:49 smithi155 bash[59495]: cluster 2023-10-09T06:15:48.609084+0000 mgr.y (mgr.25723) 311 : cluster 0 pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:49.977 INFO:teuthology.orchestra.run.smithi125.stdout:442381631503 2023-10-09T06:15:49.977 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:15:51.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:15:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:15:51] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:15:52.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:51 smithi125 bash[77299]: cluster 2023-10-09T06:15:50.609770+0000 mgr.y (mgr.25723) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:52.067 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:51 smithi125 bash[80002]: cluster 2023-10-09T06:15:50.609770+0000 mgr.y (mgr.25723) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:51 smithi155 bash[59495]: cluster 2023-10-09T06:15:50.609770+0000 mgr.y (mgr.25723) 312 : cluster 0 pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:53.073 INFO:teuthology.orchestra.run.smithi125.stdout:399431958555 2023-10-09T06:15:53.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:15:53.074 INFO:teuthology.orchestra.run.smithi125.stdout:154618822878 2023-10-09T06:15:53.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:15:53.658 INFO:teuthology.orchestra.run.smithi125.stdout:206158430413 2023-10-09T06:15:53.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:15:53.660 INFO:teuthology.orchestra.run.smithi125.stdout:420906795029 2023-10-09T06:15:53.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:15:53.895 INFO:teuthology.orchestra.run.smithi125.stdout:377957122081 2023-10-09T06:15:53.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:15:54.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:53 smithi125 bash[77299]: cluster 2023-10-09T06:15:52.610361+0000 mgr.y (mgr.25723) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:54.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:53 smithi125 bash[77299]: audit 2023-10-09T06:15:52.993655+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:54.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:53 smithi125 bash[80002]: cluster 2023-10-09T06:15:52.610361+0000 mgr.y (mgr.25723) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:54.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:53 smithi125 bash[80002]: audit 2023-10-09T06:15:52.993655+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:53 smithi155 bash[59495]: cluster 2023-10-09T06:15:52.610361+0000 mgr.y (mgr.25723) 313 : cluster 0 pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:53 smithi155 bash[59495]: audit 2023-10-09T06:15:52.993655+0000 mon.a (mon.0) 455 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:54.707 INFO:teuthology.orchestra.run.smithi125.stdout:180388626646 2023-10-09T06:15:54.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:15:54.807 INFO:teuthology.orchestra.run.smithi125.stdout:128849019112 2023-10-09T06:15:54.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:15:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:55 smithi125 bash[77299]: cluster 2023-10-09T06:15:54.611230+0000 mgr.y (mgr.25723) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:55 smithi125 bash[80002]: cluster 2023-10-09T06:15:54.611230+0000 mgr.y (mgr.25723) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:56.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:55 smithi155 bash[59495]: cluster 2023-10-09T06:15:54.611230+0000 mgr.y (mgr.25723) 314 : cluster 0 pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:57.846 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:57.847 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:57.847 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:57.847 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:57.848 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:57.850 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:57 smithi125 bash[77299]: cluster 2023-10-09T06:15:56.612009+0000 mgr.y (mgr.25723) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:57 smithi125 bash[80002]: cluster 2023-10-09T06:15:56.612009+0000 mgr.y (mgr.25723) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:58.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:57 smithi155 bash[59495]: cluster 2023-10-09T06:15:56.612009+0000 mgr.y (mgr.25723) 315 : cluster 0 pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:15:58.854 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:15:58.855 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: cluster 2023-10-09T06:15:58.612502+0000 mgr.y (mgr.25723) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:58.891439+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:58.897891+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:59.185983+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:59.193957+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:59.670718+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:59.671937+0000 mon.a (mon.0) 461 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:00.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[77299]: audit 2023-10-09T06:15:59.678478+0000 mon.a (mon.0) 462 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: cluster 2023-10-09T06:15:58.612502+0000 mgr.y (mgr.25723) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:58.891439+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:58.897891+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:59.185983+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:59.193957+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:59.670718+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:59.671937+0000 mon.a (mon.0) 461 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:00.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:15:59 smithi125 bash[80002]: audit 2023-10-09T06:15:59.678478+0000 mon.a (mon.0) 462 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: cluster 2023-10-09T06:15:58.612502+0000 mgr.y (mgr.25723) 316 : cluster 0 pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:58.891439+0000 mon.a (mon.0) 456 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:58.897891+0000 mon.a (mon.0) 457 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:59.185983+0000 mon.a (mon.0) 458 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:59.193957+0000 mon.a (mon.0) 459 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:00.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:59.670718+0000 mon.a (mon.0) 460 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:00.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:59.671937+0000 mon.a (mon.0) 461 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:00.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:15:59 smithi155 bash[59495]: audit 2023-10-09T06:15:59.678478+0000 mon.a (mon.0) 462 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:16:01.566 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:01] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:16:02.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:01 smithi125 bash[77299]: cluster 2023-10-09T06:16:00.613351+0000 mgr.y (mgr.25723) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:02.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:01 smithi125 bash[80002]: cluster 2023-10-09T06:16:00.613351+0000 mgr.y (mgr.25723) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:02.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:01 smithi155 bash[59495]: cluster 2023-10-09T06:16:00.613351+0000 mgr.y (mgr.25723) 317 : cluster 0 pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:04.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:03 smithi125 bash[80002]: cluster 2023-10-09T06:16:02.614090+0000 mgr.y (mgr.25723) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:04.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:03 smithi125 bash[80002]: audit 2023-10-09T06:16:03.558838+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:04.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:03 smithi125 bash[77299]: cluster 2023-10-09T06:16:02.614090+0000 mgr.y (mgr.25723) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:04.044 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:03 smithi125 bash[77299]: audit 2023-10-09T06:16:03.558838+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:03 smithi155 bash[59495]: cluster 2023-10-09T06:16:02.614090+0000 mgr.y (mgr.25723) 318 : cluster 0 pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:03 smithi155 bash[59495]: audit 2023-10-09T06:16:03.558838+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:06.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:05 smithi125 bash[77299]: cluster 2023-10-09T06:16:04.614758+0000 mgr.y (mgr.25723) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:06.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:05 smithi125 bash[80002]: cluster 2023-10-09T06:16:04.614758+0000 mgr.y (mgr.25723) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:06.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:05 smithi155 bash[59495]: cluster 2023-10-09T06:16:04.614758+0000 mgr.y (mgr.25723) 319 : cluster 0 pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:08.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:07 smithi125 bash[80002]: cluster 2023-10-09T06:16:06.615252+0000 mgr.y (mgr.25723) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:08.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:07 smithi125 bash[77299]: cluster 2023-10-09T06:16:06.615252+0000 mgr.y (mgr.25723) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:08.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:07 smithi155 bash[59495]: cluster 2023-10-09T06:16:06.615252+0000 mgr.y (mgr.25723) 320 : cluster 0 pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:09 smithi125 bash[77299]: cluster 2023-10-09T06:16:08.615859+0000 mgr.y (mgr.25723) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:09 smithi125 bash[80002]: cluster 2023-10-09T06:16:08.615859+0000 mgr.y (mgr.25723) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:09.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:09 smithi155 bash[59495]: cluster 2023-10-09T06:16:08.615859+0000 mgr.y (mgr.25723) 321 : cluster 0 pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:10.944 INFO:teuthology.orchestra.run.smithi125.stdout:420906795033 2023-10-09T06:16:11.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:11] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.33.4" 2023-10-09T06:16:11.825 INFO:teuthology.orchestra.run.smithi125.stdout:128849019116 2023-10-09T06:16:11.982 INFO:teuthology.orchestra.run.smithi125.stdout:442381631507 2023-10-09T06:16:11.998 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:11 smithi125 bash[77299]: cluster 2023-10-09T06:16:10.616654+0000 mgr.y (mgr.25723) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:11.998 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:11 smithi125 bash[77299]: audit 2023-10-09T06:16:10.940366+0000 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.125:0/180218229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:16:11.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:11 smithi125 bash[80002]: cluster 2023-10-09T06:16:10.616654+0000 mgr.y (mgr.25723) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:11.998 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:11 smithi125 bash[80002]: audit 2023-10-09T06:16:10.940366+0000 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.125:0/180218229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:16:12.016 INFO:teuthology.orchestra.run.smithi125.stdout:206158430418 2023-10-09T06:16:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:11 smithi155 bash[59495]: cluster 2023-10-09T06:16:10.616654+0000 mgr.y (mgr.25723) 322 : cluster 0 pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:11 smithi155 bash[59495]: audit 2023-10-09T06:16:10.940366+0000 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.125:0/180218229' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:16:12.380 INFO:teuthology.orchestra.run.smithi125.stdout:154618822883 2023-10-09T06:16:12.782 INFO:teuthology.orchestra.run.smithi125.stdout:377957122085 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[77299]: audit 2023-10-09T06:16:11.821170+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.125:0/677858502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[77299]: audit 2023-10-09T06:16:11.978103+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.125:0/2869036257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[77299]: audit 2023-10-09T06:16:12.012369+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.125:0/3429407117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[77299]: audit 2023-10-09T06:16:12.376287+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.125:0/2317873144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[80002]: audit 2023-10-09T06:16:11.821170+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.125:0/677858502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:16:13.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[80002]: audit 2023-10-09T06:16:11.978103+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.125:0/2869036257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:16:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[80002]: audit 2023-10-09T06:16:12.012369+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.125:0/3429407117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:16:13.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:12 smithi125 bash[80002]: audit 2023-10-09T06:16:12.376287+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.125:0/2317873144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:16:13.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:12 smithi155 bash[59495]: audit 2023-10-09T06:16:11.821170+0000 mon.a (mon.0) 465 : audit 0 from='client.? 172.21.15.125:0/677858502' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:16:13.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:12 smithi155 bash[59495]: audit 2023-10-09T06:16:11.978103+0000 mon.a (mon.0) 466 : audit 0 from='client.? 172.21.15.125:0/2869036257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:16:13.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:12 smithi155 bash[59495]: audit 2023-10-09T06:16:12.012369+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.125:0/3429407117' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:16:13.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:12 smithi155 bash[59495]: audit 2023-10-09T06:16:12.376287+0000 mon.a (mon.0) 467 : audit 0 from='client.? 172.21.15.125:0/2317873144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:16:13.765 INFO:teuthology.orchestra.run.smithi125.stdout:399431958560 2023-10-09T06:16:13.935 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795029 got 420906795033 for osd.2 2023-10-09T06:16:13.935 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:13 smithi125 bash[77299]: cluster 2023-10-09T06:16:12.617100+0000 mgr.y (mgr.25723) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:13 smithi125 bash[77299]: audit 2023-10-09T06:16:12.779100+0000 mon.a (mon.0) 468 : audit 0 from='client.? 172.21.15.125:0/3733836852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:16:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:13 smithi125 bash[80002]: cluster 2023-10-09T06:16:12.617100+0000 mgr.y (mgr.25723) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:13 smithi125 bash[80002]: audit 2023-10-09T06:16:12.779100+0000 mon.a (mon.0) 468 : audit 0 from='client.? 172.21.15.125:0/3733836852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:16:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:13 smithi155 bash[59495]: cluster 2023-10-09T06:16:12.617100+0000 mgr.y (mgr.25723) 323 : cluster 0 pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:13 smithi155 bash[59495]: audit 2023-10-09T06:16:12.779100+0000 mon.a (mon.0) 468 : audit 0 from='client.? 172.21.15.125:0/3733836852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:16:14.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:14 smithi125 bash[80002]: audit 2023-10-09T06:16:13.761170+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.125:0/1830115868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:16:14.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:14 smithi125 bash[77299]: audit 2023-10-09T06:16:13.761170+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.125:0/1830115868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:16:15.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:14 smithi155 bash[59495]: audit 2023-10-09T06:16:13.761170+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.125:0/1830115868' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:16:15.346 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631503 got 442381631507 for osd.3 2023-10-09T06:16:15.347 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:16.090 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:15 smithi125 bash[80002]: cluster 2023-10-09T06:16:14.617709+0000 mgr.y (mgr.25723) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:16.091 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:15 smithi125 bash[77299]: cluster 2023-10-09T06:16:14.617709+0000 mgr.y (mgr.25723) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:16.092 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430413 got 206158430418 for osd.7 2023-10-09T06:16:16.092 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:16.137 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019112 got 128849019116 for osd.4 2023-10-09T06:16:16.137 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:15 smithi155 bash[59495]: cluster 2023-10-09T06:16:14.617709+0000 mgr.y (mgr.25723) 324 : cluster 0 pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:16.630 INFO:teuthology.orchestra.run.smithi125.stdout:180388626651 2023-10-09T06:16:16.668 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122081 got 377957122085 for osd.0 2023-10-09T06:16:16.668 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:17.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:16 smithi125 bash[80002]: audit 2023-10-09T06:16:16.626157+0000 mon.a (mon.0) 470 : audit 0 from='client.? 172.21.15.125:0/4182251244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:16:17.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:16 smithi125 bash[77299]: audit 2023-10-09T06:16:16.626157+0000 mon.a (mon.0) 470 : audit 0 from='client.? 172.21.15.125:0/4182251244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:16:17.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:16 smithi155 bash[59495]: audit 2023-10-09T06:16:16.626157+0000 mon.a (mon.0) 470 : audit 0 from='client.? 172.21.15.125:0/4182251244' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:16:17.687 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958555 got 399431958560 for osd.1 2023-10-09T06:16:17.699 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:17.710 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822878 got 154618822883 for osd.5 2023-10-09T06:16:17.710 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:17.779 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626646 got 180388626651 for osd.6 2023-10-09T06:16:17.858 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:17.858 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:16:17.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:16:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:17 smithi125 bash[77299]: cluster 2023-10-09T06:16:16.618077+0000 mgr.y (mgr.25723) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:18.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:17 smithi125 bash[80002]: cluster 2023-10-09T06:16:16.618077+0000 mgr.y (mgr.25723) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:18.157 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:17 smithi155 bash[59495]: cluster 2023-10-09T06:16:16.618077+0000 mgr.y (mgr.25723) 325 : cluster 0 pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:19 smithi125 bash[77299]: audit 2023-10-09T06:16:18.559541+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:19.914 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:19 smithi125 bash[77299]: cluster 2023-10-09T06:16:18.618858+0000 mgr.y (mgr.25723) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:19.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:19 smithi125 bash[80002]: audit 2023-10-09T06:16:18.559541+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:19.915 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:19 smithi125 bash[80002]: cluster 2023-10-09T06:16:18.618858+0000 mgr.y (mgr.25723) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:19.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:19 smithi155 bash[59495]: audit 2023-10-09T06:16:18.559541+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:19.915 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:19 smithi155 bash[59495]: cluster 2023-10-09T06:16:18.618858+0000 mgr.y (mgr.25723) 326 : cluster 0 pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:21.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:21] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:16:22.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:21 smithi125 bash[77299]: cluster 2023-10-09T06:16:20.619704+0000 mgr.y (mgr.25723) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:22.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:21 smithi125 bash[80002]: cluster 2023-10-09T06:16:20.619704+0000 mgr.y (mgr.25723) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:22.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:21 smithi155 bash[59495]: cluster 2023-10-09T06:16:20.619704+0000 mgr.y (mgr.25723) 327 : cluster 0 pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:22.635 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:24.087 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:23 smithi125 bash[77299]: cluster 2023-10-09T06:16:22.620644+0000 mgr.y (mgr.25723) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:24.087 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:23 smithi125 bash[80002]: cluster 2023-10-09T06:16:22.620644+0000 mgr.y (mgr.25723) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:24.149 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:23 smithi155 bash[59495]: cluster 2023-10-09T06:16:22.620644+0000 mgr.y (mgr.25723) 328 : cluster 0 pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:25.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:25 smithi125 bash[77299]: cluster 2023-10-09T06:16:24.621388+0000 mgr.y (mgr.25723) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:25.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:25 smithi125 bash[80002]: cluster 2023-10-09T06:16:24.621388+0000 mgr.y (mgr.25723) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:25.906 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:16:25.910 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:16:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:25 smithi155 bash[59495]: cluster 2023-10-09T06:16:24.621388+0000 mgr.y (mgr.25723) 329 : cluster 0 pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:26.824 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":266,"stamp":"2023-10-09T06:16:24.620910+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1340996,"kb_used_data":22532,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620220,"statfs":{"total":767960285184,"available":766587105280,"internally_reserved":0,"allocated":23072768,"data_stored":19011447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1350009967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004060"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664027+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:14:41.664027+0000","last_peered":"2023-10-09T06:14:41.664027+0000","last_clean":"2023-10-09T06:14:41.664027+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:14:41.664027+0000","last_undegraded":"2023-10-09T06:14:41.664027+0000","last_fullsized":"2023-10-09T06:14:41.664027+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.666619+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:14:42.666619+0000","last_peered":"2023-10-09T06:14:42.666619+0000","last_clean":"2023-10-09T06:14:42.666619+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:14:42.666619+0000","last_undegraded":"2023-10-09T06:14:42.666619+0000","last_fullsized":"2023-10-09T06:14:42.666619+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680070+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:14:42.680070+0000","last_peered":"2023-10-09T06:14:42.680070+0000","last_clean":"2023-10-09T06:14:42.680070+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:14:42.680070+0000","last_undegraded":"2023-10-09T06:14:42.680070+0000","last_fullsized":"2023-10-09T06:14:42.680070+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688599+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:14:42.688599+0000","last_peered":"2023-10-09T06:14:42.688599+0000","last_clean":"2023-10-09T06:14:42.688599+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:14:42.688599+0000","last_undegraded":"2023-10-09T06:14:42.688599+0000","last_fullsized":"2023-10-09T06:14:42.688599+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.687434+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:14:42.687434+0000","last_peered":"2023-10-09T06:14:42.687434+0000","last_clean":"2023-10-09T06:14:42.687434+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:14:42.687434+0000","last_undegraded":"2023-10-09T06:14:42.687434+0000","last_fullsized":"2023-10-09T06:14:42.687434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.668343+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:14:41.668343+0000","last_peered":"2023-10-09T06:14:41.668343+0000","last_clean":"2023-10-09T06:14:41.668343+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:14:41.668343+0000","last_undegraded":"2023-10-09T06:14:41.668343+0000","last_fullsized":"2023-10-09T06:14:41.668343+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688026+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:14:42.688026+0000","last_peered":"2023-10-09T06:14:42.688026+0000","last_clean":"2023-10-09T06:14:42.688026+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:14:42.688026+0000","last_undegraded":"2023-10-09T06:14:42.688026+0000","last_fullsized":"2023-10-09T06:14:42.688026+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667440+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:14:41.667440+0000","last_peered":"2023-10-09T06:14:41.667440+0000","last_clean":"2023-10-09T06:14:41.667440+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:14:41.667440+0000","last_undegraded":"2023-10-09T06:14:41.667440+0000","last_fullsized":"2023-10-09T06:14:41.667440+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682776+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:14:42.682776+0000","last_peered":"2023-10-09T06:14:42.682776+0000","last_clean":"2023-10-09T06:14:42.682776+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:14:42.682776+0000","last_undegraded":"2023-10-09T06:14:42.682776+0000","last_fullsized":"2023-10-09T06:14:42.682776+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668373+0000","last_change":"2023-10-09T06:13:14.649836+0000","last_active":"2023-10-09T06:14:42.668373+0000","last_peered":"2023-10-09T06:14:42.668373+0000","last_clean":"2023-10-09T06:14:42.668373+0000","last_became_active":"2023-10-09T06:13:14.649052+0000","last_became_peered":"2023-10-09T06:13:14.649052+0000","last_unstale":"2023-10-09T06:14:42.668373+0000","last_undegraded":"2023-10-09T06:14:42.668373+0000","last_fullsized":"2023-10-09T06:14:42.668373+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:00.038681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667778+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:14:41.667778+0000","last_peered":"2023-10-09T06:14:41.667778+0000","last_clean":"2023-10-09T06:14:41.667778+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:14:41.667778+0000","last_undegraded":"2023-10-09T06:14:41.667778+0000","last_fullsized":"2023-10-09T06:14:41.667778+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668192+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:14:42.668192+0000","last_peered":"2023-10-09T06:14:42.668192+0000","last_clean":"2023-10-09T06:14:42.668192+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:14:42.668192+0000","last_undegraded":"2023-10-09T06:14:42.668192+0000","last_fullsized":"2023-10-09T06:14:42.668192+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":722,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661488+0000","last_change":"2023-10-09T06:14:12.896068+0000","last_active":"2023-10-09T06:14:41.661488+0000","last_peered":"2023-10-09T06:14:41.661488+0000","last_clean":"2023-10-09T06:14:41.661488+0000","last_became_active":"2023-10-09T06:14:12.895488+0000","last_became_peered":"2023-10-09T06:14:12.895488+0000","last_unstale":"2023-10-09T06:14:41.661488+0000","last_undegraded":"2023-10-09T06:14:41.661488+0000","last_fullsized":"2023-10-09T06:14:41.661488+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682247+0000","last_change":"2023-10-09T06:14:42.682247+0000","last_active":"2023-10-09T06:14:42.682247+0000","last_peered":"2023-10-09T06:14:42.682247+0000","last_clean":"2023-10-09T06:14:42.682247+0000","last_became_active":"2023-10-09T06:14:42.681892+0000","last_became_peered":"2023-10-09T06:14:42.681892+0000","last_unstale":"2023-10-09T06:14:42.682247+0000","last_undegraded":"2023-10-09T06:14:42.682247+0000","last_fullsized":"2023-10-09T06:14:42.682247+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:34.655393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":405,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675316+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:14:42.675316+0000","last_peered":"2023-10-09T06:14:42.675316+0000","last_clean":"2023-10-09T06:14:42.675316+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:14:42.675316+0000","last_undegraded":"2023-10-09T06:14:42.675316+0000","last_fullsized":"2023-10-09T06:14:42.675316+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684347+0000","last_change":"2023-10-09T06:14:42.684347+0000","last_active":"2023-10-09T06:14:42.684347+0000","last_peered":"2023-10-09T06:14:42.684347+0000","last_clean":"2023-10-09T06:14:42.684347+0000","last_became_active":"2023-10-09T06:14:42.683791+0000","last_became_peered":"2023-10-09T06:14:42.683791+0000","last_unstale":"2023-10-09T06:14:42.684347+0000","last_undegraded":"2023-10-09T06:14:42.684347+0000","last_fullsized":"2023-10-09T06:14:42.684347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:46.220918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668111+0000","last_change":"2023-10-09T06:13:14.649857+0000","last_active":"2023-10-09T06:14:42.668111+0000","last_peered":"2023-10-09T06:14:42.668111+0000","last_clean":"2023-10-09T06:14:42.668111+0000","last_became_active":"2023-10-09T06:13:14.649499+0000","last_became_peered":"2023-10-09T06:13:14.649499+0000","last_unstale":"2023-10-09T06:14:42.668111+0000","last_undegraded":"2023-10-09T06:14:42.668111+0000","last_fullsized":"2023-10-09T06:14:42.668111+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:01.624443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676962+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:14:42.676962+0000","last_peered":"2023-10-09T06:14:42.676962+0000","last_clean":"2023-10-09T06:14:42.676962+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:14:42.676962+0000","last_undegraded":"2023-10-09T06:14:42.676962+0000","last_fullsized":"2023-10-09T06:14:42.676962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":397,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672428+0000","last_change":"2023-10-09T06:13:14.650725+0000","last_active":"2023-10-09T06:14:41.672428+0000","last_peered":"2023-10-09T06:14:41.672428+0000","last_clean":"2023-10-09T06:14:41.672428+0000","last_became_active":"2023-10-09T06:13:14.649943+0000","last_became_peered":"2023-10-09T06:13:14.649943+0000","last_unstale":"2023-10-09T06:14:41.672428+0000","last_undegraded":"2023-10-09T06:14:41.672428+0000","last_fullsized":"2023-10-09T06:14:41.672428+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668292+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:14:42.668292+0000","last_peered":"2023-10-09T06:14:42.668292+0000","last_clean":"2023-10-09T06:14:42.668292+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:14:42.668292+0000","last_undegraded":"2023-10-09T06:14:42.668292+0000","last_fullsized":"2023-10-09T06:14:42.668292+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680125+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:14:42.680125+0000","last_peered":"2023-10-09T06:14:42.680125+0000","last_clean":"2023-10-09T06:14:42.680125+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:14:42.680125+0000","last_undegraded":"2023-10-09T06:14:42.680125+0000","last_fullsized":"2023-10-09T06:14:42.680125+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":417,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671389+0000","last_change":"2023-10-09T06:13:43.436803+0000","last_active":"2023-10-09T06:14:41.671389+0000","last_peered":"2023-10-09T06:14:41.671389+0000","last_clean":"2023-10-09T06:14:41.671389+0000","last_became_active":"2023-10-09T06:13:43.435952+0000","last_became_peered":"2023-10-09T06:13:43.435952+0000","last_unstale":"2023-10-09T06:14:41.671389+0000","last_undegraded":"2023-10-09T06:14:41.671389+0000","last_fullsized":"2023-10-09T06:14:41.671389+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661963+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:14:41.661963+0000","last_peered":"2023-10-09T06:14:41.661963+0000","last_clean":"2023-10-09T06:14:41.661963+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:14:41.661963+0000","last_undegraded":"2023-10-09T06:14:41.661963+0000","last_fullsized":"2023-10-09T06:14:41.661963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688699+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:14:42.688699+0000","last_peered":"2023-10-09T06:14:42.688699+0000","last_clean":"2023-10-09T06:14:42.688699+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:14:42.688699+0000","last_undegraded":"2023-10-09T06:14:42.688699+0000","last_fullsized":"2023-10-09T06:14:42.688699+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":305,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671590+0000","last_change":"2023-10-09T06:13:14.652137+0000","last_active":"2023-10-09T06:14:41.671590+0000","last_peered":"2023-10-09T06:14:41.671590+0000","last_clean":"2023-10-09T06:14:41.671590+0000","last_became_active":"2023-10-09T06:13:14.650321+0000","last_became_peered":"2023-10-09T06:13:14.650321+0000","last_unstale":"2023-10-09T06:14:41.671590+0000","last_undegraded":"2023-10-09T06:14:41.671590+0000","last_fullsized":"2023-10-09T06:14:41.671590+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":370,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.665991+0000","last_change":"2023-10-09T06:14:12.894599+0000","last_active":"2023-10-09T06:14:41.665991+0000","last_peered":"2023-10-09T06:14:41.665991+0000","last_clean":"2023-10-09T06:14:41.665991+0000","last_became_active":"2023-10-09T06:14:12.894067+0000","last_became_peered":"2023-10-09T06:14:12.894067+0000","last_unstale":"2023-10-09T06:14:41.665991+0000","last_undegraded":"2023-10-09T06:14:41.665991+0000","last_fullsized":"2023-10-09T06:14:41.665991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676067+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:14:42.676067+0000","last_peered":"2023-10-09T06:14:42.676067+0000","last_clean":"2023-10-09T06:14:42.676067+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:14:42.676067+0000","last_undegraded":"2023-10-09T06:14:42.676067+0000","last_fullsized":"2023-10-09T06:14:42.676067+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675555+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:14:42.675555+0000","last_peered":"2023-10-09T06:14:42.675555+0000","last_clean":"2023-10-09T06:14:42.675555+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:14:42.675555+0000","last_undegraded":"2023-10-09T06:14:42.675555+0000","last_fullsized":"2023-10-09T06:14:42.675555+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":460,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682883+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:14:42.682883+0000","last_peered":"2023-10-09T06:14:42.682883+0000","last_clean":"2023-10-09T06:14:42.682883+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:14:42.682883+0000","last_undegraded":"2023-10-09T06:14:42.682883+0000","last_fullsized":"2023-10-09T06:14:42.682883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686512+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:14:42.686512+0000","last_peered":"2023-10-09T06:14:42.686512+0000","last_clean":"2023-10-09T06:14:42.686512+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:14:42.686512+0000","last_undegraded":"2023-10-09T06:14:42.686512+0000","last_fullsized":"2023-10-09T06:14:42.686512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669168+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:14:41.669168+0000","last_peered":"2023-10-09T06:14:41.669168+0000","last_clean":"2023-10-09T06:14:41.669168+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:14:41.669168+0000","last_undegraded":"2023-10-09T06:14:41.669168+0000","last_fullsized":"2023-10-09T06:14:41.669168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":685,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664444+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:14:41.664444+0000","last_peered":"2023-10-09T06:14:41.664444+0000","last_clean":"2023-10-09T06:14:41.664444+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:14:41.664444+0000","last_undegraded":"2023-10-09T06:14:41.664444+0000","last_fullsized":"2023-10-09T06:14:41.664444+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674979+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:14:42.674979+0000","last_peered":"2023-10-09T06:14:42.674979+0000","last_clean":"2023-10-09T06:14:42.674979+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:14:42.674979+0000","last_undegraded":"2023-10-09T06:14:42.674979+0000","last_fullsized":"2023-10-09T06:14:42.674979+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.690962+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:14:42.690962+0000","last_peered":"2023-10-09T06:14:42.690962+0000","last_clean":"2023-10-09T06:14:42.690962+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:14:42.690962+0000","last_undegraded":"2023-10-09T06:14:42.690962+0000","last_fullsized":"2023-10-09T06:14:42.690962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":859,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676911+0000","last_change":"2023-10-09T06:13:43.431318+0000","last_active":"2023-10-09T06:14:42.676911+0000","last_peered":"2023-10-09T06:14:42.676911+0000","last_clean":"2023-10-09T06:14:42.676911+0000","last_became_active":"2023-10-09T06:13:43.430651+0000","last_became_peered":"2023-10-09T06:13:43.430651+0000","last_unstale":"2023-10-09T06:14:42.676911+0000","last_undegraded":"2023-10-09T06:14:42.676911+0000","last_fullsized":"2023-10-09T06:14:42.676911+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.401600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.685730+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:14:42.685730+0000","last_peered":"2023-10-09T06:14:42.685730+0000","last_clean":"2023-10-09T06:14:42.685730+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:14:42.685730+0000","last_undegraded":"2023-10-09T06:14:42.685730+0000","last_fullsized":"2023-10-09T06:14:42.685730+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681524+0000","last_change":"2023-10-09T06:14:42.681524+0000","last_active":"2023-10-09T06:14:42.681524+0000","last_peered":"2023-10-09T06:14:42.681524+0000","last_clean":"2023-10-09T06:14:42.681524+0000","last_became_active":"2023-10-09T06:14:42.681183+0000","last_became_peered":"2023-10-09T06:14:42.681183+0000","last_unstale":"2023-10-09T06:14:42.681524+0000","last_undegraded":"2023-10-09T06:14:42.681524+0000","last_fullsized":"2023-10-09T06:14:42.681524+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:37.001075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666609+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:14:41.666609+0000","last_peered":"2023-10-09T06:14:41.666609+0000","last_clean":"2023-10-09T06:14:41.666609+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:14:41.666609+0000","last_undegraded":"2023-10-09T06:14:41.666609+0000","last_fullsized":"2023-10-09T06:14:41.666609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667896+0000","last_change":"2023-10-09T06:14:12.892112+0000","last_active":"2023-10-09T06:14:41.667896+0000","last_peered":"2023-10-09T06:14:41.667896+0000","last_clean":"2023-10-09T06:14:41.667896+0000","last_became_active":"2023-10-09T06:14:12.891310+0000","last_became_peered":"2023-10-09T06:14:12.891310+0000","last_unstale":"2023-10-09T06:14:41.667896+0000","last_undegraded":"2023-10-09T06:14:41.667896+0000","last_fullsized":"2023-10-09T06:14:41.667896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666855+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:14:41.666855+0000","last_peered":"2023-10-09T06:14:41.666855+0000","last_clean":"2023-10-09T06:14:41.666855+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:14:41.666855+0000","last_undegraded":"2023-10-09T06:14:41.666855+0000","last_fullsized":"2023-10-09T06:14:41.666855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.662347+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:14:41.662347+0000","last_peered":"2023-10-09T06:14:41.662347+0000","last_clean":"2023-10-09T06:14:41.662347+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:14:41.662347+0000","last_undegraded":"2023-10-09T06:14:41.662347+0000","last_fullsized":"2023-10-09T06:14:41.662347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689434+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:14:42.689434+0000","last_peered":"2023-10-09T06:14:42.689434+0000","last_clean":"2023-10-09T06:14:42.689434+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:14:42.689434+0000","last_undegraded":"2023-10-09T06:14:42.689434+0000","last_fullsized":"2023-10-09T06:14:42.689434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680748+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:14:42.680748+0000","last_peered":"2023-10-09T06:14:42.680748+0000","last_clean":"2023-10-09T06:14:42.680748+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:14:42.680748+0000","last_undegraded":"2023-10-09T06:14:42.680748+0000","last_fullsized":"2023-10-09T06:14:42.680748+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668252+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:14:42.668252+0000","last_peered":"2023-10-09T06:14:42.668252+0000","last_clean":"2023-10-09T06:14:42.668252+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:14:42.668252+0000","last_undegraded":"2023-10-09T06:14:42.668252+0000","last_fullsized":"2023-10-09T06:14:42.668252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691189+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:14:42.691189+0000","last_peered":"2023-10-09T06:14:42.691189+0000","last_clean":"2023-10-09T06:14:42.691189+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:14:42.691189+0000","last_undegraded":"2023-10-09T06:14:42.691189+0000","last_fullsized":"2023-10-09T06:14:42.691189+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661806+0000","last_change":"2023-10-09T06:13:43.443807+0000","last_active":"2023-10-09T06:14:41.661806+0000","last_peered":"2023-10-09T06:14:41.661806+0000","last_clean":"2023-10-09T06:14:41.661806+0000","last_became_active":"2023-10-09T06:13:43.443290+0000","last_became_peered":"2023-10-09T06:13:43.443290+0000","last_unstale":"2023-10-09T06:14:41.661806+0000","last_undegraded":"2023-10-09T06:14:41.661806+0000","last_fullsized":"2023-10-09T06:14:41.661806+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672583+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:14:41.672583+0000","last_peered":"2023-10-09T06:14:41.672583+0000","last_clean":"2023-10-09T06:14:41.672583+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:14:41.672583+0000","last_undegraded":"2023-10-09T06:14:41.672583+0000","last_fullsized":"2023-10-09T06:14:41.672583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688520+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:14:42.688520+0000","last_peered":"2023-10-09T06:14:42.688520+0000","last_clean":"2023-10-09T06:14:42.688520+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:14:42.688520+0000","last_undegraded":"2023-10-09T06:14:42.688520+0000","last_fullsized":"2023-10-09T06:14:42.688520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672036+0000","last_change":"2023-10-09T06:13:43.436130+0000","last_active":"2023-10-09T06:14:41.672036+0000","last_peered":"2023-10-09T06:14:41.672036+0000","last_clean":"2023-10-09T06:14:41.672036+0000","last_became_active":"2023-10-09T06:13:43.435753+0000","last_became_peered":"2023-10-09T06:13:43.435753+0000","last_unstale":"2023-10-09T06:14:41.672036+0000","last_undegraded":"2023-10-09T06:14:41.672036+0000","last_fullsized":"2023-10-09T06:14:41.672036+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664290+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:14:41.664290+0000","last_peered":"2023-10-09T06:14:41.664290+0000","last_clean":"2023-10-09T06:14:41.664290+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:14:41.664290+0000","last_undegraded":"2023-10-09T06:14:41.664290+0000","last_fullsized":"2023-10-09T06:14:41.664290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671979+0000","last_change":"2023-10-09T06:13:14.651888+0000","last_active":"2023-10-09T06:14:41.671979+0000","last_peered":"2023-10-09T06:14:41.671979+0000","last_clean":"2023-10-09T06:14:41.671979+0000","last_became_active":"2023-10-09T06:13:14.651493+0000","last_became_peered":"2023-10-09T06:13:14.651493+0000","last_unstale":"2023-10-09T06:14:41.671979+0000","last_undegraded":"2023-10-09T06:14:41.671979+0000","last_fullsized":"2023-10-09T06:14:41.671979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.684353+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.683803+0000","last_became_peered":"2023-10-09T06:14:42.683803+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:55.893179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661070+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:14:41.661070+0000","last_peered":"2023-10-09T06:14:41.661070+0000","last_clean":"2023-10-09T06:14:41.661070+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:14:41.661070+0000","last_undegraded":"2023-10-09T06:14:41.661070+0000","last_fullsized":"2023-10-09T06:14:41.661070+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681933+0000","last_change":"2023-10-09T06:14:42.681256+0000","last_active":"2023-10-09T06:14:42.681933+0000","last_peered":"2023-10-09T06:14:42.681933+0000","last_clean":"2023-10-09T06:14:42.681933+0000","last_became_active":"2023-10-09T06:14:42.680856+0000","last_became_peered":"2023-10-09T06:14:42.680856+0000","last_unstale":"2023-10-09T06:14:42.681933+0000","last_undegraded":"2023-10-09T06:14:42.681933+0000","last_fullsized":"2023-10-09T06:14:42.681933+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:17.803358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.670656+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:14:41.670656+0000","last_peered":"2023-10-09T06:14:41.670656+0000","last_clean":"2023-10-09T06:14:41.670656+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:14:41.670656+0000","last_undegraded":"2023-10-09T06:14:41.670656+0000","last_fullsized":"2023-10-09T06:14:41.670656+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691481+0000","last_change":"2023-10-09T06:14:42.691481+0000","last_active":"2023-10-09T06:14:42.691481+0000","last_peered":"2023-10-09T06:14:42.691481+0000","last_clean":"2023-10-09T06:14:42.691481+0000","last_became_active":"2023-10-09T06:14:42.690934+0000","last_became_peered":"2023-10-09T06:14:42.690934+0000","last_unstale":"2023-10-09T06:14:42.691481+0000","last_undegraded":"2023-10-09T06:14:42.691481+0000","last_fullsized":"2023-10-09T06:14:42.691481+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683010+0000","last_change":"2023-10-09T06:14:42.682720+0000","last_active":"2023-10-09T06:14:42.683010+0000","last_peered":"2023-10-09T06:14:42.683010+0000","last_clean":"2023-10-09T06:14:42.683010+0000","last_became_active":"2023-10-09T06:14:42.682336+0000","last_became_peered":"2023-10-09T06:14:42.682336+0000","last_unstale":"2023-10-09T06:14:42.683010+0000","last_undegraded":"2023-10-09T06:14:42.683010+0000","last_fullsized":"2023-10-09T06:14:42.683010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:17.620826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683136+0000","last_change":"2023-10-09T06:14:42.683136+0000","last_active":"2023-10-09T06:14:42.683136+0000","last_peered":"2023-10-09T06:14:42.683136+0000","last_clean":"2023-10-09T06:14:42.683136+0000","last_became_active":"2023-10-09T06:14:42.682686+0000","last_became_peered":"2023-10-09T06:14:42.682686+0000","last_unstale":"2023-10-09T06:14:42.683136+0000","last_undegraded":"2023-10-09T06:14:42.683136+0000","last_fullsized":"2023-10-09T06:14:42.683136+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:09.556766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671277+0000","last_change":"2023-10-09T06:14:12.890850+0000","last_active":"2023-10-09T06:14:41.671277+0000","last_peered":"2023-10-09T06:14:41.671277+0000","last_clean":"2023-10-09T06:14:41.671277+0000","last_became_active":"2023-10-09T06:14:12.890340+0000","last_became_peered":"2023-10-09T06:14:12.890340+0000","last_unstale":"2023-10-09T06:14:41.671277+0000","last_undegraded":"2023-10-09T06:14:41.671277+0000","last_fullsized":"2023-10-09T06:14:41.671277+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669746+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:14:41.669746+0000","last_peered":"2023-10-09T06:14:41.669746+0000","last_clean":"2023-10-09T06:14:41.669746+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:14:41.669746+0000","last_undegraded":"2023-10-09T06:14:41.669746+0000","last_fullsized":"2023-10-09T06:14:41.669746+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":76,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667591+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:14:41.667591+0000","last_peered":"2023-10-09T06:14:41.667591+0000","last_clean":"2023-10-09T06:14:41.667591+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:14:41.667591+0000","last_undegraded":"2023-10-09T06:14:41.667591+0000","last_fullsized":"2023-10-09T06:14:41.667591+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674873+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:14:42.674873+0000","last_peered":"2023-10-09T06:14:42.674873+0000","last_clean":"2023-10-09T06:14:42.674873+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:14:42.674873+0000","last_undegraded":"2023-10-09T06:14:42.674873+0000","last_fullsized":"2023-10-09T06:14:42.674873+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689505+0000","last_change":"2023-10-09T06:14:42.689505+0000","last_active":"2023-10-09T06:14:42.689505+0000","last_peered":"2023-10-09T06:14:42.689505+0000","last_clean":"2023-10-09T06:14:42.689505+0000","last_became_active":"2023-10-09T06:14:42.688999+0000","last_became_peered":"2023-10-09T06:14:42.688999+0000","last_unstale":"2023-10-09T06:14:42.689505+0000","last_undegraded":"2023-10-09T06:14:42.689505+0000","last_fullsized":"2023-10-09T06:14:42.689505+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688566+0000","last_change":"2023-10-09T06:14:42.688566+0000","last_active":"2023-10-09T06:14:42.688566+0000","last_peered":"2023-10-09T06:14:42.688566+0000","last_clean":"2023-10-09T06:14:42.688566+0000","last_became_active":"2023-10-09T06:14:42.687716+0000","last_became_peered":"2023-10-09T06:14:42.687716+0000","last_unstale":"2023-10-09T06:14:42.688566+0000","last_undegraded":"2023-10-09T06:14:42.688566+0000","last_fullsized":"2023-10-09T06:14:42.688566+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:16.554385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664423+0000","last_change":"2023-10-09T06:13:43.433440+0000","last_active":"2023-10-09T06:14:41.664423+0000","last_peered":"2023-10-09T06:14:41.664423+0000","last_clean":"2023-10-09T06:14:41.664423+0000","last_became_active":"2023-10-09T06:13:43.432996+0000","last_became_peered":"2023-10-09T06:13:43.432996+0000","last_unstale":"2023-10-09T06:14:41.664423+0000","last_undegraded":"2023-10-09T06:14:41.664423+0000","last_fullsized":"2023-10-09T06:14:41.664423+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688101+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:14:42.688101+0000","last_peered":"2023-10-09T06:14:42.688101+0000","last_clean":"2023-10-09T06:14:42.688101+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:14:42.688101+0000","last_undegraded":"2023-10-09T06:14:42.688101+0000","last_fullsized":"2023-10-09T06:14:42.688101+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684738+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:14:42.684738+0000","last_peered":"2023-10-09T06:14:42.684738+0000","last_clean":"2023-10-09T06:14:42.684738+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:14:42.684738+0000","last_undegraded":"2023-10-09T06:14:42.684738+0000","last_fullsized":"2023-10-09T06:14:42.684738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661265+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:14:41.661265+0000","last_peered":"2023-10-09T06:14:41.661265+0000","last_clean":"2023-10-09T06:14:41.661265+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:14:41.661265+0000","last_undegraded":"2023-10-09T06:14:41.661265+0000","last_fullsized":"2023-10-09T06:14:41.661265+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.679929+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:14:42.679929+0000","last_peered":"2023-10-09T06:14:42.679929+0000","last_clean":"2023-10-09T06:14:42.679929+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:14:42.679929+0000","last_undegraded":"2023-10-09T06:14:42.679929+0000","last_fullsized":"2023-10-09T06:14:42.679929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682509+0000","last_change":"2023-10-09T06:14:42.681947+0000","last_active":"2023-10-09T06:14:42.682509+0000","last_peered":"2023-10-09T06:14:42.682509+0000","last_clean":"2023-10-09T06:14:42.682509+0000","last_became_active":"2023-10-09T06:14:42.681620+0000","last_became_peered":"2023-10-09T06:14:42.681620+0000","last_unstale":"2023-10-09T06:14:42.682509+0000","last_undegraded":"2023-10-09T06:14:42.682509+0000","last_fullsized":"2023-10-09T06:14:42.682509+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:39.518730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686269+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:14:42.686269+0000","last_peered":"2023-10-09T06:14:42.686269+0000","last_clean":"2023-10-09T06:14:42.686269+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:14:42.686269+0000","last_undegraded":"2023-10-09T06:14:42.686269+0000","last_fullsized":"2023-10-09T06:14:42.686269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681459+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:14:42.681459+0000","last_peered":"2023-10-09T06:14:42.681459+0000","last_clean":"2023-10-09T06:14:42.681459+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:14:42.681459+0000","last_undegraded":"2023-10-09T06:14:42.681459+0000","last_fullsized":"2023-10-09T06:14:42.681459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666343+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:14:41.666343+0000","last_peered":"2023-10-09T06:14:41.666343+0000","last_clean":"2023-10-09T06:14:41.666343+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:14:41.666343+0000","last_undegraded":"2023-10-09T06:14:41.666343+0000","last_fullsized":"2023-10-09T06:14:41.666343+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430421,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302832,"kb_used_data":1640,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442320,"statfs":{"total":95995035648,"available":95684935680,"internally_reserved":0,"allocated":1679360,"data_stored":1171733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.61499999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.86699999999999999,"15min":0.85699999999999998},"min":{"1min":0.60999999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1419999999999999,"5min":1.224,"15min":1.224},"last":0.997}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.88500000000000001,"15min":0.86499999999999999},"min":{"1min":0.71099999999999997,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.2629999999999999,"5min":1.276,"15min":1.276},"last":1.2869999999999999},{"interface":"front","average":{"1min":1.073,"5min":0.94099999999999995,"15min":0.91900000000000004},"min":{"1min":0.83799999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.0880000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92800000000000005,"15min":0.93500000000000005},"min":{"1min":0.67100000000000004,"5min":0.625,"15min":0.625},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.997},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.2689999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.1679999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:43 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":1.052},{"interface":"front","average":{"1min":1.151,"5min":1.151,"15min":1.151},"min":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.92100000000000004,"15min":0.91900000000000004},"min":{"1min":0.70999999999999996,"5min":0.498,"15min":0.436},"max":{"1min":1.21,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.97099999999999997,"15min":0.93700000000000006},"min":{"1min":0.74399999999999999,"5min":0.52700000000000002,"15min":0.49399999999999999},"max":{"1min":1.2789999999999999,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.143}]},{"osd":5,"last update":"Mon Oct 9 06:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.014,"15min":0.98499999999999999},"min":{"1min":0.73699999999999999,"5min":0.61099999999999999,"15min":0.48399999999999999},"max":{"1min":1.4039999999999999,"5min":1.4419999999999999,"15min":1.5449999999999999},"last":1.2450000000000001},{"interface":"front","average":{"1min":0.995,"5min":1.0329999999999999,"15min":1.0509999999999999},"min":{"1min":0.86199999999999999,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3160000000000001,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.268}]},{"osd":6,"last update":"Mon Oct 9 06:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.0720000000000001,"15min":1.089},"min":{"1min":0.77800000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.228,"5min":1.577,"15min":1.579},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.0920000000000001,"15min":1.0569999999999999},"min":{"1min":0.78000000000000003,"5min":0.69599999999999995,"15min":0.53700000000000003},"max":{"1min":1.2869999999999999,"5min":1.423,"15min":1.466},"last":1.222}]}]},{"osd":6,"up_from":42,"seq":180388626653,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302816,"kb_used_data":1624,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442336,"statfs":{"total":95995035648,"available":95684952064,"internally_reserved":0,"allocated":1662976,"data_stored":1170976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.90100000000000002,"15min":0.88500000000000001},"min":{"1min":0.69199999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2569999999999999,"5min":1.274,"15min":1.274},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.872,"5min":0.91400000000000003,"15min":0.93600000000000005},"min":{"1min":0.60399999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2030000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.1220000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.871,"15min":0.86199999999999999},"min":{"1min":0.625,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.89300000000000002,"15min":0.89100000000000001},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 06:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.97799999999999998,"15min":0.97399999999999998},"min":{"1min":0.753,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.008,"5min":0.94799999999999995,"15min":0.93799999999999994},"min":{"1min":0.751,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.92600000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.071,"15min":1.071},"min":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":1.048}]},{"osd":4,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.93500000000000005,"15min":0.92200000000000004},"min":{"1min":0.74299999999999999,"5min":0.63800000000000001,"15min":0.45000000000000001},"max":{"1min":1.24,"5min":1.391,"15min":1.5289999999999999},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.96799999999999997},"min":{"1min":0.71399999999999997,"5min":0.58999999999999997,"15min":0.47399999999999998},"max":{"1min":1.2789999999999999,"5min":1.3240000000000001,"15min":1.4590000000000001},"last":1.2150000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.0289999999999999,"15min":1.0089999999999999},"min":{"1min":0.89400000000000002,"5min":0.63200000000000001,"15min":0.54200000000000004},"max":{"1min":1.401,"5min":1.401,"15min":1.708},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0509999999999999,"15min":1.0369999999999999},"min":{"1min":0.72999999999999998,"5min":0.59899999999999998,"15min":0.50700000000000001},"max":{"1min":1.353,"5min":1.4019999999999999,"15min":1.4590000000000001},"last":1.1559999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.093,"15min":1.0740000000000001},"min":{"1min":0.94899999999999995,"5min":0.71899999999999997,"15min":0.47399999999999998},"max":{"1min":1.3600000000000001,"5min":1.3600000000000001,"15min":1.629},"last":1.254},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.125,"15min":1.1080000000000001},"min":{"1min":0.86699999999999999,"5min":0.72599999999999998,"15min":0.44700000000000001},"max":{"1min":1.411,"5min":1.5189999999999999,"15min":1.5329999999999999},"last":1.1850000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958562,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":586594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":32301456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.71599999999999997,"15min":0.72199999999999998},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.4370000000000001,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.69099999999999995,"15min":0.68899999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.98499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.67400000000000004,"15min":0.67600000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.90300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":1.405},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.8959999999999999,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.88300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.98699999999999999,"15min":0.97099999999999997},"min":{"1min":0.80000000000000004,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.5669999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.0029999999999999,"15min":1.0089999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":1.464}]},{"osd":5,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0329999999999999,"15min":1.034},"min":{"1min":0.70699999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":2.0150000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":1.7390000000000001},{"interface":"front","average":{"1min":1.145,"5min":1.054,"15min":1.0389999999999999},"min":{"1min":0.82299999999999995,"5min":0.76900000000000002,"15min":0.76900000000000002},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":1.494}]},{"osd":6,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.075,"15min":1.077},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.9670000000000001,"5min":1.9670000000000001,"15min":1.9670000000000001},"last":1.603},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.139,"15min":1.131},"min":{"1min":0.94699999999999995,"5min":0.91400000000000003,"15min":0.91400000000000003},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.877}]},{"osd":7,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.1419999999999999,"15min":1.129},"min":{"1min":0.88700000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"max":{"1min":1.625,"5min":1.625,"15min":1.625},"last":1.786},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.163,"15min":1.1759999999999999},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":1.3720000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122088,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37332,"kb_used_data":5772,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707820,"statfs":{"total":95995035648,"available":95956807680,"internally_reserved":0,"allocated":5910528,"data_stored":5371872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":32300288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61699999999999999,"15min":0.59899999999999998},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.73399999999999999,"15min":0.71599999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.113,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":1.1619999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.746},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71299999999999997,"15min":0.71899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":1.2450000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.93100000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:15:24 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.93300000000000005,"15min":0.93200000000000005},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.99399999999999999,"15min":1.0009999999999999},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.2430000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.98499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:15:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.03,"15min":1.042},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.1479999999999999,"5min":1.575,"15min":1.575},"last":1.095},{"interface":"front","average":{"1min":0.995,"5min":1.052,"15min":1.0620000000000001},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.2649999999999999,"5min":1.472,"15min":1.472},"last":1.3320000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.071,"15min":1.081},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.343,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.4119999999999999},{"interface":"front","average":{"1min":1.03,"5min":1.016,"15min":1.014},"min":{"1min":0.77700000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.3460000000000001,"5min":1.524,"15min":1.524},"last":1.133}]},{"osd":7,"last update":"Mon Oct 9 06:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.1459999999999999,"15min":1.1539999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.413,"5min":1.667,"15min":1.667},"last":1.458},{"interface":"front","average":{"1min":1.109,"5min":1.119,"15min":1.121},"min":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"max":{"1min":1.4419999999999999,"5min":1.7,"15min":1.7},"last":1.206}]}]},{"osd":2,"up_from":98,"seq":420906795036,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36664,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708488,"statfs":{"total":95995035648,"available":95957491712,"internally_reserved":0,"allocated":5292032,"data_stored":4777297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":32232382},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.65600000000000003,"15min":0.64900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.028,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69899999999999995,"15min":0.69999999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0209999999999999,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.92500000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.873},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.64300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.96499999999999997,"15min":0.95499999999999996},"min":{"1min":0.61899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.264,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.93999999999999995,"15min":0.92700000000000005},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.99399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.006,"15min":1.016},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.3560000000000001,"5min":1.488,"15min":1.488},"last":0.752},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.65700000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.22,"5min":1.27,"15min":1.27},"last":0.81399999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.78800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3720000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3089999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.96199999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.091,"15min":1.095},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.389,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0940000000000001,"15min":1.0960000000000001},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.484,"5min":1.54,"15min":1.54},"last":1.099}]}]},{"osd":3,"up_from":103,"seq":442381631510,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32424,"kb_used_data":5152,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93712728,"statfs":{"total":95995035648,"available":95961833472,"internally_reserved":0,"allocated":5275648,"data_stored":4775252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":27914785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.746},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.78300000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.089,"5min":1.089,"15min":1.089},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.88900000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":1.127},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.70799999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.1739999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.026}]},{"osd":6,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":1.3979999999999999},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.577,"5min":1.577,"15min":1.577},"last":0.91600000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"min":{"1min":0.752,"5min":0.752,"15min":0.752},"max":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.097},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"min":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.3169999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019119,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.96799999999999997,"15min":0.96399999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.198,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0129999999999999,"15min":1.0249999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.69999999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.96199999999999997,"15min":0.95999999999999996},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.006,"5min":0.98999999999999999,"15min":0.98799999999999999},"min":{"1min":0.73599999999999999,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.0569999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.91200000000000003,"15min":0.90900000000000003},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.88600000000000001,"15min":0.88400000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.47,"5min":1.47,"15min":1.47},"last":0.67200000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.99099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:15:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.91800000000000004,"15min":0.90000000000000002},"min":{"1min":0.65300000000000002,"5min":0.59999999999999998,"15min":0.437},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.869},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.004,"5min":0.98599999999999999,"15min":0.97099999999999997},"min":{"1min":0.58099999999999996,"5min":0.55700000000000005,"15min":0.50600000000000001},"max":{"1min":1.4390000000000001,"5min":1.466,"15min":1.9890000000000001},"last":0.92200000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:15:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.008,"15min":1.0049999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.48799999999999999},"max":{"1min":1.2929999999999999,"5min":1.6180000000000001,"15min":2.0859999999999999},"last":1.02},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.006,"15min":0.98999999999999999},"min":{"1min":0.76200000000000001,"5min":0.53000000000000003,"15min":0.436},"max":{"1min":1.367,"5min":1.411,"15min":1.7290000000000001},"last":1.198}]},{"osd":7,"last update":"Mon Oct 9 06:15:28 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.0720000000000001,"15min":1.0549999999999999},"min":{"1min":0.72299999999999998,"5min":0.57199999999999995,"15min":0.52200000000000002},"max":{"1min":1.431,"5min":1.556,"15min":2.056},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.077,"5min":1.0640000000000001,"15min":1.054},"min":{"1min":0.79200000000000004,"5min":0.60499999999999998,"15min":0.504},"max":{"1min":1.528,"5min":1.5720000000000001,"15min":2.024},"last":1.117}]}]},{"osd":5,"up_from":36,"seq":154618822886,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298132,"kb_used_data":1036,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447020,"statfs":{"total":95995035648,"available":95689748480,"internally_reserved":0,"allocated":1060864,"data_stored":576428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.93200000000000005,"15min":0.94499999999999995},"min":{"1min":0.57499999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.89400000000000002,"15min":0.89200000000000002},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1830000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.1200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94199999999999995,"15min":0.94399999999999995},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.90800000000000003,"15min":0.90500000000000003},"min":{"1min":0.70099999999999996,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.96299999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.93899999999999995,"15min":0.92300000000000004},"min":{"1min":0.72599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.024,"15min":1.026},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.071}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.056},{"interface":"front","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.1759999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.90100000000000002,"15min":0.90900000000000003},"min":{"1min":0.64900000000000002,"5min":0.52600000000000002,"15min":0.42499999999999999},"max":{"1min":1.1459999999999999,"5min":1.399,"15min":1.526},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.996,"15min":0.98699999999999999},"min":{"1min":0.68300000000000005,"5min":0.55800000000000005,"15min":0.49299999999999999},"max":{"1min":1.387,"5min":1.4059999999999999,"15min":1.845},"last":0.752}]},{"osd":6,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.97399999999999998,"15min":0.99099999999999999},"min":{"1min":0.67400000000000004,"5min":0.54300000000000004,"15min":0.51200000000000001},"max":{"1min":1.2330000000000001,"5min":1.3460000000000001,"15min":1.4830000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.149,"5min":1.1060000000000001,"15min":1.071},"min":{"1min":0.76600000000000001,"5min":0.67000000000000004,"15min":0.51400000000000001},"max":{"1min":1.4730000000000001,"5min":1.591,"15min":1.9019999999999999},"last":1.153}]},{"osd":7,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.133,"15min":1.0960000000000001},"min":{"1min":0.94299999999999995,"5min":0.63700000000000001,"15min":0.52400000000000002},"max":{"1min":1.5860000000000001,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1399999999999999,"15min":1.125},"min":{"1min":0.95299999999999996,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":2.3969999999999998,"15min":2.3969999999999998},"last":1.024}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:16:26.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:16:27.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:26 smithi125 bash[77299]: audit 2023-10-09T06:16:25.883030+0000 mgr.y (mgr.25723) 330 : audit 0 from='client.44600 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:27.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:26 smithi125 bash[80002]: audit 2023-10-09T06:16:25.883030+0000 mgr.y (mgr.25723) 330 : audit 0 from='client.44600 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:27.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:26 smithi155 bash[59495]: audit 2023-10-09T06:16:25.883030+0000 mgr.y (mgr.25723) 330 : audit 0 from='client.44600 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:28.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:27 smithi125 bash[77299]: cluster 2023-10-09T06:16:26.622183+0000 mgr.y (mgr.25723) 331 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:27 smithi125 bash[80002]: cluster 2023-10-09T06:16:26.622183+0000 mgr.y (mgr.25723) 331 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:28.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:27 smithi155 bash[59495]: cluster 2023-10-09T06:16:26.622183+0000 mgr.y (mgr.25723) 331 : cluster 0 pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:29 smithi125 bash[77299]: cluster 2023-10-09T06:16:28.622936+0000 mgr.y (mgr.25723) 332 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:29 smithi125 bash[80002]: cluster 2023-10-09T06:16:28.622936+0000 mgr.y (mgr.25723) 332 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:29.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:29 smithi155 bash[59495]: cluster 2023-10-09T06:16:28.622936+0000 mgr.y (mgr.25723) 332 : cluster 0 pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:31.607 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:31.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:31] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.33.4" 2023-10-09T06:16:31.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:31 smithi125 bash[77299]: cluster 2023-10-09T06:16:30.623828+0000 mgr.y (mgr.25723) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:31.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:31 smithi125 bash[80002]: cluster 2023-10-09T06:16:30.623828+0000 mgr.y (mgr.25723) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:31 smithi155 bash[59495]: cluster 2023-10-09T06:16:30.623828+0000 mgr.y (mgr.25723) 333 : cluster 0 pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:33 smithi125 bash[77299]: cluster 2023-10-09T06:16:32.624639+0000 mgr.y (mgr.25723) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:33 smithi125 bash[77299]: audit 2023-10-09T06:16:33.559831+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:33 smithi125 bash[80002]: cluster 2023-10-09T06:16:32.624639+0000 mgr.y (mgr.25723) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:34.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:33 smithi125 bash[80002]: audit 2023-10-09T06:16:33.559831+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:34.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:33 smithi155 bash[59495]: cluster 2023-10-09T06:16:32.624639+0000 mgr.y (mgr.25723) 334 : cluster 0 pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:34.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:33 smithi155 bash[59495]: audit 2023-10-09T06:16:33.559831+0000 mon.a (mon.0) 472 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:35.037 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:16:35.040 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:16:35.881 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":271,"stamp":"2023-10-09T06:16:34.624921+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1340996,"kb_used_data":22532,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620220,"statfs":{"total":767960285184,"available":766587105280,"internally_reserved":0,"allocated":23072768,"data_stored":19011447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1350009967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004878"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664027+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:14:41.664027+0000","last_peered":"2023-10-09T06:14:41.664027+0000","last_clean":"2023-10-09T06:14:41.664027+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:14:41.664027+0000","last_undegraded":"2023-10-09T06:14:41.664027+0000","last_fullsized":"2023-10-09T06:14:41.664027+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.666619+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:14:42.666619+0000","last_peered":"2023-10-09T06:14:42.666619+0000","last_clean":"2023-10-09T06:14:42.666619+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:14:42.666619+0000","last_undegraded":"2023-10-09T06:14:42.666619+0000","last_fullsized":"2023-10-09T06:14:42.666619+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680070+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:14:42.680070+0000","last_peered":"2023-10-09T06:14:42.680070+0000","last_clean":"2023-10-09T06:14:42.680070+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:14:42.680070+0000","last_undegraded":"2023-10-09T06:14:42.680070+0000","last_fullsized":"2023-10-09T06:14:42.680070+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688599+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:14:42.688599+0000","last_peered":"2023-10-09T06:14:42.688599+0000","last_clean":"2023-10-09T06:14:42.688599+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:14:42.688599+0000","last_undegraded":"2023-10-09T06:14:42.688599+0000","last_fullsized":"2023-10-09T06:14:42.688599+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.687434+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:14:42.687434+0000","last_peered":"2023-10-09T06:14:42.687434+0000","last_clean":"2023-10-09T06:14:42.687434+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:14:42.687434+0000","last_undegraded":"2023-10-09T06:14:42.687434+0000","last_fullsized":"2023-10-09T06:14:42.687434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.668343+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:14:41.668343+0000","last_peered":"2023-10-09T06:14:41.668343+0000","last_clean":"2023-10-09T06:14:41.668343+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:14:41.668343+0000","last_undegraded":"2023-10-09T06:14:41.668343+0000","last_fullsized":"2023-10-09T06:14:41.668343+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688026+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:14:42.688026+0000","last_peered":"2023-10-09T06:14:42.688026+0000","last_clean":"2023-10-09T06:14:42.688026+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:14:42.688026+0000","last_undegraded":"2023-10-09T06:14:42.688026+0000","last_fullsized":"2023-10-09T06:14:42.688026+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667440+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:14:41.667440+0000","last_peered":"2023-10-09T06:14:41.667440+0000","last_clean":"2023-10-09T06:14:41.667440+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:14:41.667440+0000","last_undegraded":"2023-10-09T06:14:41.667440+0000","last_fullsized":"2023-10-09T06:14:41.667440+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682776+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:14:42.682776+0000","last_peered":"2023-10-09T06:14:42.682776+0000","last_clean":"2023-10-09T06:14:42.682776+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:14:42.682776+0000","last_undegraded":"2023-10-09T06:14:42.682776+0000","last_fullsized":"2023-10-09T06:14:42.682776+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668373+0000","last_change":"2023-10-09T06:13:14.649836+0000","last_active":"2023-10-09T06:14:42.668373+0000","last_peered":"2023-10-09T06:14:42.668373+0000","last_clean":"2023-10-09T06:14:42.668373+0000","last_became_active":"2023-10-09T06:13:14.649052+0000","last_became_peered":"2023-10-09T06:13:14.649052+0000","last_unstale":"2023-10-09T06:14:42.668373+0000","last_undegraded":"2023-10-09T06:14:42.668373+0000","last_fullsized":"2023-10-09T06:14:42.668373+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:00.038681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667778+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:14:41.667778+0000","last_peered":"2023-10-09T06:14:41.667778+0000","last_clean":"2023-10-09T06:14:41.667778+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:14:41.667778+0000","last_undegraded":"2023-10-09T06:14:41.667778+0000","last_fullsized":"2023-10-09T06:14:41.667778+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668192+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:14:42.668192+0000","last_peered":"2023-10-09T06:14:42.668192+0000","last_clean":"2023-10-09T06:14:42.668192+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:14:42.668192+0000","last_undegraded":"2023-10-09T06:14:42.668192+0000","last_fullsized":"2023-10-09T06:14:42.668192+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":722,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661488+0000","last_change":"2023-10-09T06:14:12.896068+0000","last_active":"2023-10-09T06:14:41.661488+0000","last_peered":"2023-10-09T06:14:41.661488+0000","last_clean":"2023-10-09T06:14:41.661488+0000","last_became_active":"2023-10-09T06:14:12.895488+0000","last_became_peered":"2023-10-09T06:14:12.895488+0000","last_unstale":"2023-10-09T06:14:41.661488+0000","last_undegraded":"2023-10-09T06:14:41.661488+0000","last_fullsized":"2023-10-09T06:14:41.661488+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682247+0000","last_change":"2023-10-09T06:14:42.682247+0000","last_active":"2023-10-09T06:14:42.682247+0000","last_peered":"2023-10-09T06:14:42.682247+0000","last_clean":"2023-10-09T06:14:42.682247+0000","last_became_active":"2023-10-09T06:14:42.681892+0000","last_became_peered":"2023-10-09T06:14:42.681892+0000","last_unstale":"2023-10-09T06:14:42.682247+0000","last_undegraded":"2023-10-09T06:14:42.682247+0000","last_fullsized":"2023-10-09T06:14:42.682247+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:34.655393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":405,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675316+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:14:42.675316+0000","last_peered":"2023-10-09T06:14:42.675316+0000","last_clean":"2023-10-09T06:14:42.675316+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:14:42.675316+0000","last_undegraded":"2023-10-09T06:14:42.675316+0000","last_fullsized":"2023-10-09T06:14:42.675316+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684347+0000","last_change":"2023-10-09T06:14:42.684347+0000","last_active":"2023-10-09T06:14:42.684347+0000","last_peered":"2023-10-09T06:14:42.684347+0000","last_clean":"2023-10-09T06:14:42.684347+0000","last_became_active":"2023-10-09T06:14:42.683791+0000","last_became_peered":"2023-10-09T06:14:42.683791+0000","last_unstale":"2023-10-09T06:14:42.684347+0000","last_undegraded":"2023-10-09T06:14:42.684347+0000","last_fullsized":"2023-10-09T06:14:42.684347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:46.220918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668111+0000","last_change":"2023-10-09T06:13:14.649857+0000","last_active":"2023-10-09T06:14:42.668111+0000","last_peered":"2023-10-09T06:14:42.668111+0000","last_clean":"2023-10-09T06:14:42.668111+0000","last_became_active":"2023-10-09T06:13:14.649499+0000","last_became_peered":"2023-10-09T06:13:14.649499+0000","last_unstale":"2023-10-09T06:14:42.668111+0000","last_undegraded":"2023-10-09T06:14:42.668111+0000","last_fullsized":"2023-10-09T06:14:42.668111+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:01.624443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676962+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:14:42.676962+0000","last_peered":"2023-10-09T06:14:42.676962+0000","last_clean":"2023-10-09T06:14:42.676962+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:14:42.676962+0000","last_undegraded":"2023-10-09T06:14:42.676962+0000","last_fullsized":"2023-10-09T06:14:42.676962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":397,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672428+0000","last_change":"2023-10-09T06:13:14.650725+0000","last_active":"2023-10-09T06:14:41.672428+0000","last_peered":"2023-10-09T06:14:41.672428+0000","last_clean":"2023-10-09T06:14:41.672428+0000","last_became_active":"2023-10-09T06:13:14.649943+0000","last_became_peered":"2023-10-09T06:13:14.649943+0000","last_unstale":"2023-10-09T06:14:41.672428+0000","last_undegraded":"2023-10-09T06:14:41.672428+0000","last_fullsized":"2023-10-09T06:14:41.672428+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668292+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:14:42.668292+0000","last_peered":"2023-10-09T06:14:42.668292+0000","last_clean":"2023-10-09T06:14:42.668292+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:14:42.668292+0000","last_undegraded":"2023-10-09T06:14:42.668292+0000","last_fullsized":"2023-10-09T06:14:42.668292+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680125+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:14:42.680125+0000","last_peered":"2023-10-09T06:14:42.680125+0000","last_clean":"2023-10-09T06:14:42.680125+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:14:42.680125+0000","last_undegraded":"2023-10-09T06:14:42.680125+0000","last_fullsized":"2023-10-09T06:14:42.680125+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":417,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671389+0000","last_change":"2023-10-09T06:13:43.436803+0000","last_active":"2023-10-09T06:14:41.671389+0000","last_peered":"2023-10-09T06:14:41.671389+0000","last_clean":"2023-10-09T06:14:41.671389+0000","last_became_active":"2023-10-09T06:13:43.435952+0000","last_became_peered":"2023-10-09T06:13:43.435952+0000","last_unstale":"2023-10-09T06:14:41.671389+0000","last_undegraded":"2023-10-09T06:14:41.671389+0000","last_fullsized":"2023-10-09T06:14:41.671389+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661963+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:14:41.661963+0000","last_peered":"2023-10-09T06:14:41.661963+0000","last_clean":"2023-10-09T06:14:41.661963+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:14:41.661963+0000","last_undegraded":"2023-10-09T06:14:41.661963+0000","last_fullsized":"2023-10-09T06:14:41.661963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688699+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:14:42.688699+0000","last_peered":"2023-10-09T06:14:42.688699+0000","last_clean":"2023-10-09T06:14:42.688699+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:14:42.688699+0000","last_undegraded":"2023-10-09T06:14:42.688699+0000","last_fullsized":"2023-10-09T06:14:42.688699+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":305,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671590+0000","last_change":"2023-10-09T06:13:14.652137+0000","last_active":"2023-10-09T06:14:41.671590+0000","last_peered":"2023-10-09T06:14:41.671590+0000","last_clean":"2023-10-09T06:14:41.671590+0000","last_became_active":"2023-10-09T06:13:14.650321+0000","last_became_peered":"2023-10-09T06:13:14.650321+0000","last_unstale":"2023-10-09T06:14:41.671590+0000","last_undegraded":"2023-10-09T06:14:41.671590+0000","last_fullsized":"2023-10-09T06:14:41.671590+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":370,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.665991+0000","last_change":"2023-10-09T06:14:12.894599+0000","last_active":"2023-10-09T06:14:41.665991+0000","last_peered":"2023-10-09T06:14:41.665991+0000","last_clean":"2023-10-09T06:14:41.665991+0000","last_became_active":"2023-10-09T06:14:12.894067+0000","last_became_peered":"2023-10-09T06:14:12.894067+0000","last_unstale":"2023-10-09T06:14:41.665991+0000","last_undegraded":"2023-10-09T06:14:41.665991+0000","last_fullsized":"2023-10-09T06:14:41.665991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676067+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:14:42.676067+0000","last_peered":"2023-10-09T06:14:42.676067+0000","last_clean":"2023-10-09T06:14:42.676067+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:14:42.676067+0000","last_undegraded":"2023-10-09T06:14:42.676067+0000","last_fullsized":"2023-10-09T06:14:42.676067+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675555+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:14:42.675555+0000","last_peered":"2023-10-09T06:14:42.675555+0000","last_clean":"2023-10-09T06:14:42.675555+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:14:42.675555+0000","last_undegraded":"2023-10-09T06:14:42.675555+0000","last_fullsized":"2023-10-09T06:14:42.675555+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":460,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682883+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:14:42.682883+0000","last_peered":"2023-10-09T06:14:42.682883+0000","last_clean":"2023-10-09T06:14:42.682883+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:14:42.682883+0000","last_undegraded":"2023-10-09T06:14:42.682883+0000","last_fullsized":"2023-10-09T06:14:42.682883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686512+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:14:42.686512+0000","last_peered":"2023-10-09T06:14:42.686512+0000","last_clean":"2023-10-09T06:14:42.686512+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:14:42.686512+0000","last_undegraded":"2023-10-09T06:14:42.686512+0000","last_fullsized":"2023-10-09T06:14:42.686512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669168+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:14:41.669168+0000","last_peered":"2023-10-09T06:14:41.669168+0000","last_clean":"2023-10-09T06:14:41.669168+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:14:41.669168+0000","last_undegraded":"2023-10-09T06:14:41.669168+0000","last_fullsized":"2023-10-09T06:14:41.669168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":685,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664444+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:14:41.664444+0000","last_peered":"2023-10-09T06:14:41.664444+0000","last_clean":"2023-10-09T06:14:41.664444+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:14:41.664444+0000","last_undegraded":"2023-10-09T06:14:41.664444+0000","last_fullsized":"2023-10-09T06:14:41.664444+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674979+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:14:42.674979+0000","last_peered":"2023-10-09T06:14:42.674979+0000","last_clean":"2023-10-09T06:14:42.674979+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:14:42.674979+0000","last_undegraded":"2023-10-09T06:14:42.674979+0000","last_fullsized":"2023-10-09T06:14:42.674979+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.690962+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:14:42.690962+0000","last_peered":"2023-10-09T06:14:42.690962+0000","last_clean":"2023-10-09T06:14:42.690962+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:14:42.690962+0000","last_undegraded":"2023-10-09T06:14:42.690962+0000","last_fullsized":"2023-10-09T06:14:42.690962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":859,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676911+0000","last_change":"2023-10-09T06:13:43.431318+0000","last_active":"2023-10-09T06:14:42.676911+0000","last_peered":"2023-10-09T06:14:42.676911+0000","last_clean":"2023-10-09T06:14:42.676911+0000","last_became_active":"2023-10-09T06:13:43.430651+0000","last_became_peered":"2023-10-09T06:13:43.430651+0000","last_unstale":"2023-10-09T06:14:42.676911+0000","last_undegraded":"2023-10-09T06:14:42.676911+0000","last_fullsized":"2023-10-09T06:14:42.676911+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.401600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.685730+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:14:42.685730+0000","last_peered":"2023-10-09T06:14:42.685730+0000","last_clean":"2023-10-09T06:14:42.685730+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:14:42.685730+0000","last_undegraded":"2023-10-09T06:14:42.685730+0000","last_fullsized":"2023-10-09T06:14:42.685730+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681524+0000","last_change":"2023-10-09T06:14:42.681524+0000","last_active":"2023-10-09T06:14:42.681524+0000","last_peered":"2023-10-09T06:14:42.681524+0000","last_clean":"2023-10-09T06:14:42.681524+0000","last_became_active":"2023-10-09T06:14:42.681183+0000","last_became_peered":"2023-10-09T06:14:42.681183+0000","last_unstale":"2023-10-09T06:14:42.681524+0000","last_undegraded":"2023-10-09T06:14:42.681524+0000","last_fullsized":"2023-10-09T06:14:42.681524+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:37.001075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666609+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:14:41.666609+0000","last_peered":"2023-10-09T06:14:41.666609+0000","last_clean":"2023-10-09T06:14:41.666609+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:14:41.666609+0000","last_undegraded":"2023-10-09T06:14:41.666609+0000","last_fullsized":"2023-10-09T06:14:41.666609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667896+0000","last_change":"2023-10-09T06:14:12.892112+0000","last_active":"2023-10-09T06:14:41.667896+0000","last_peered":"2023-10-09T06:14:41.667896+0000","last_clean":"2023-10-09T06:14:41.667896+0000","last_became_active":"2023-10-09T06:14:12.891310+0000","last_became_peered":"2023-10-09T06:14:12.891310+0000","last_unstale":"2023-10-09T06:14:41.667896+0000","last_undegraded":"2023-10-09T06:14:41.667896+0000","last_fullsized":"2023-10-09T06:14:41.667896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666855+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:14:41.666855+0000","last_peered":"2023-10-09T06:14:41.666855+0000","last_clean":"2023-10-09T06:14:41.666855+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:14:41.666855+0000","last_undegraded":"2023-10-09T06:14:41.666855+0000","last_fullsized":"2023-10-09T06:14:41.666855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.662347+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:14:41.662347+0000","last_peered":"2023-10-09T06:14:41.662347+0000","last_clean":"2023-10-09T06:14:41.662347+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:14:41.662347+0000","last_undegraded":"2023-10-09T06:14:41.662347+0000","last_fullsized":"2023-10-09T06:14:41.662347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689434+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:14:42.689434+0000","last_peered":"2023-10-09T06:14:42.689434+0000","last_clean":"2023-10-09T06:14:42.689434+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:14:42.689434+0000","last_undegraded":"2023-10-09T06:14:42.689434+0000","last_fullsized":"2023-10-09T06:14:42.689434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680748+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:14:42.680748+0000","last_peered":"2023-10-09T06:14:42.680748+0000","last_clean":"2023-10-09T06:14:42.680748+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:14:42.680748+0000","last_undegraded":"2023-10-09T06:14:42.680748+0000","last_fullsized":"2023-10-09T06:14:42.680748+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668252+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:14:42.668252+0000","last_peered":"2023-10-09T06:14:42.668252+0000","last_clean":"2023-10-09T06:14:42.668252+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:14:42.668252+0000","last_undegraded":"2023-10-09T06:14:42.668252+0000","last_fullsized":"2023-10-09T06:14:42.668252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691189+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:14:42.691189+0000","last_peered":"2023-10-09T06:14:42.691189+0000","last_clean":"2023-10-09T06:14:42.691189+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:14:42.691189+0000","last_undegraded":"2023-10-09T06:14:42.691189+0000","last_fullsized":"2023-10-09T06:14:42.691189+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661806+0000","last_change":"2023-10-09T06:13:43.443807+0000","last_active":"2023-10-09T06:14:41.661806+0000","last_peered":"2023-10-09T06:14:41.661806+0000","last_clean":"2023-10-09T06:14:41.661806+0000","last_became_active":"2023-10-09T06:13:43.443290+0000","last_became_peered":"2023-10-09T06:13:43.443290+0000","last_unstale":"2023-10-09T06:14:41.661806+0000","last_undegraded":"2023-10-09T06:14:41.661806+0000","last_fullsized":"2023-10-09T06:14:41.661806+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672583+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:14:41.672583+0000","last_peered":"2023-10-09T06:14:41.672583+0000","last_clean":"2023-10-09T06:14:41.672583+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:14:41.672583+0000","last_undegraded":"2023-10-09T06:14:41.672583+0000","last_fullsized":"2023-10-09T06:14:41.672583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688520+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:14:42.688520+0000","last_peered":"2023-10-09T06:14:42.688520+0000","last_clean":"2023-10-09T06:14:42.688520+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:14:42.688520+0000","last_undegraded":"2023-10-09T06:14:42.688520+0000","last_fullsized":"2023-10-09T06:14:42.688520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672036+0000","last_change":"2023-10-09T06:13:43.436130+0000","last_active":"2023-10-09T06:14:41.672036+0000","last_peered":"2023-10-09T06:14:41.672036+0000","last_clean":"2023-10-09T06:14:41.672036+0000","last_became_active":"2023-10-09T06:13:43.435753+0000","last_became_peered":"2023-10-09T06:13:43.435753+0000","last_unstale":"2023-10-09T06:14:41.672036+0000","last_undegraded":"2023-10-09T06:14:41.672036+0000","last_fullsized":"2023-10-09T06:14:41.672036+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664290+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:14:41.664290+0000","last_peered":"2023-10-09T06:14:41.664290+0000","last_clean":"2023-10-09T06:14:41.664290+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:14:41.664290+0000","last_undegraded":"2023-10-09T06:14:41.664290+0000","last_fullsized":"2023-10-09T06:14:41.664290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671979+0000","last_change":"2023-10-09T06:13:14.651888+0000","last_active":"2023-10-09T06:14:41.671979+0000","last_peered":"2023-10-09T06:14:41.671979+0000","last_clean":"2023-10-09T06:14:41.671979+0000","last_became_active":"2023-10-09T06:13:14.651493+0000","last_became_peered":"2023-10-09T06:13:14.651493+0000","last_unstale":"2023-10-09T06:14:41.671979+0000","last_undegraded":"2023-10-09T06:14:41.671979+0000","last_fullsized":"2023-10-09T06:14:41.671979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.684353+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.683803+0000","last_became_peered":"2023-10-09T06:14:42.683803+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:55.893179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661070+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:14:41.661070+0000","last_peered":"2023-10-09T06:14:41.661070+0000","last_clean":"2023-10-09T06:14:41.661070+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:14:41.661070+0000","last_undegraded":"2023-10-09T06:14:41.661070+0000","last_fullsized":"2023-10-09T06:14:41.661070+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681933+0000","last_change":"2023-10-09T06:14:42.681256+0000","last_active":"2023-10-09T06:14:42.681933+0000","last_peered":"2023-10-09T06:14:42.681933+0000","last_clean":"2023-10-09T06:14:42.681933+0000","last_became_active":"2023-10-09T06:14:42.680856+0000","last_became_peered":"2023-10-09T06:14:42.680856+0000","last_unstale":"2023-10-09T06:14:42.681933+0000","last_undegraded":"2023-10-09T06:14:42.681933+0000","last_fullsized":"2023-10-09T06:14:42.681933+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:17.803358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.670656+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:14:41.670656+0000","last_peered":"2023-10-09T06:14:41.670656+0000","last_clean":"2023-10-09T06:14:41.670656+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:14:41.670656+0000","last_undegraded":"2023-10-09T06:14:41.670656+0000","last_fullsized":"2023-10-09T06:14:41.670656+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691481+0000","last_change":"2023-10-09T06:14:42.691481+0000","last_active":"2023-10-09T06:14:42.691481+0000","last_peered":"2023-10-09T06:14:42.691481+0000","last_clean":"2023-10-09T06:14:42.691481+0000","last_became_active":"2023-10-09T06:14:42.690934+0000","last_became_peered":"2023-10-09T06:14:42.690934+0000","last_unstale":"2023-10-09T06:14:42.691481+0000","last_undegraded":"2023-10-09T06:14:42.691481+0000","last_fullsized":"2023-10-09T06:14:42.691481+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683010+0000","last_change":"2023-10-09T06:14:42.682720+0000","last_active":"2023-10-09T06:14:42.683010+0000","last_peered":"2023-10-09T06:14:42.683010+0000","last_clean":"2023-10-09T06:14:42.683010+0000","last_became_active":"2023-10-09T06:14:42.682336+0000","last_became_peered":"2023-10-09T06:14:42.682336+0000","last_unstale":"2023-10-09T06:14:42.683010+0000","last_undegraded":"2023-10-09T06:14:42.683010+0000","last_fullsized":"2023-10-09T06:14:42.683010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:17.620826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683136+0000","last_change":"2023-10-09T06:14:42.683136+0000","last_active":"2023-10-09T06:14:42.683136+0000","last_peered":"2023-10-09T06:14:42.683136+0000","last_clean":"2023-10-09T06:14:42.683136+0000","last_became_active":"2023-10-09T06:14:42.682686+0000","last_became_peered":"2023-10-09T06:14:42.682686+0000","last_unstale":"2023-10-09T06:14:42.683136+0000","last_undegraded":"2023-10-09T06:14:42.683136+0000","last_fullsized":"2023-10-09T06:14:42.683136+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:09.556766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671277+0000","last_change":"2023-10-09T06:14:12.890850+0000","last_active":"2023-10-09T06:14:41.671277+0000","last_peered":"2023-10-09T06:14:41.671277+0000","last_clean":"2023-10-09T06:14:41.671277+0000","last_became_active":"2023-10-09T06:14:12.890340+0000","last_became_peered":"2023-10-09T06:14:12.890340+0000","last_unstale":"2023-10-09T06:14:41.671277+0000","last_undegraded":"2023-10-09T06:14:41.671277+0000","last_fullsized":"2023-10-09T06:14:41.671277+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669746+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:14:41.669746+0000","last_peered":"2023-10-09T06:14:41.669746+0000","last_clean":"2023-10-09T06:14:41.669746+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:14:41.669746+0000","last_undegraded":"2023-10-09T06:14:41.669746+0000","last_fullsized":"2023-10-09T06:14:41.669746+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":76,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667591+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:14:41.667591+0000","last_peered":"2023-10-09T06:14:41.667591+0000","last_clean":"2023-10-09T06:14:41.667591+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:14:41.667591+0000","last_undegraded":"2023-10-09T06:14:41.667591+0000","last_fullsized":"2023-10-09T06:14:41.667591+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674873+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:14:42.674873+0000","last_peered":"2023-10-09T06:14:42.674873+0000","last_clean":"2023-10-09T06:14:42.674873+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:14:42.674873+0000","last_undegraded":"2023-10-09T06:14:42.674873+0000","last_fullsized":"2023-10-09T06:14:42.674873+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689505+0000","last_change":"2023-10-09T06:14:42.689505+0000","last_active":"2023-10-09T06:14:42.689505+0000","last_peered":"2023-10-09T06:14:42.689505+0000","last_clean":"2023-10-09T06:14:42.689505+0000","last_became_active":"2023-10-09T06:14:42.688999+0000","last_became_peered":"2023-10-09T06:14:42.688999+0000","last_unstale":"2023-10-09T06:14:42.689505+0000","last_undegraded":"2023-10-09T06:14:42.689505+0000","last_fullsized":"2023-10-09T06:14:42.689505+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688566+0000","last_change":"2023-10-09T06:14:42.688566+0000","last_active":"2023-10-09T06:14:42.688566+0000","last_peered":"2023-10-09T06:14:42.688566+0000","last_clean":"2023-10-09T06:14:42.688566+0000","last_became_active":"2023-10-09T06:14:42.687716+0000","last_became_peered":"2023-10-09T06:14:42.687716+0000","last_unstale":"2023-10-09T06:14:42.688566+0000","last_undegraded":"2023-10-09T06:14:42.688566+0000","last_fullsized":"2023-10-09T06:14:42.688566+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:16.554385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664423+0000","last_change":"2023-10-09T06:13:43.433440+0000","last_active":"2023-10-09T06:14:41.664423+0000","last_peered":"2023-10-09T06:14:41.664423+0000","last_clean":"2023-10-09T06:14:41.664423+0000","last_became_active":"2023-10-09T06:13:43.432996+0000","last_became_peered":"2023-10-09T06:13:43.432996+0000","last_unstale":"2023-10-09T06:14:41.664423+0000","last_undegraded":"2023-10-09T06:14:41.664423+0000","last_fullsized":"2023-10-09T06:14:41.664423+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688101+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:14:42.688101+0000","last_peered":"2023-10-09T06:14:42.688101+0000","last_clean":"2023-10-09T06:14:42.688101+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:14:42.688101+0000","last_undegraded":"2023-10-09T06:14:42.688101+0000","last_fullsized":"2023-10-09T06:14:42.688101+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684738+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:14:42.684738+0000","last_peered":"2023-10-09T06:14:42.684738+0000","last_clean":"2023-10-09T06:14:42.684738+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:14:42.684738+0000","last_undegraded":"2023-10-09T06:14:42.684738+0000","last_fullsized":"2023-10-09T06:14:42.684738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661265+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:14:41.661265+0000","last_peered":"2023-10-09T06:14:41.661265+0000","last_clean":"2023-10-09T06:14:41.661265+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:14:41.661265+0000","last_undegraded":"2023-10-09T06:14:41.661265+0000","last_fullsized":"2023-10-09T06:14:41.661265+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.679929+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:14:42.679929+0000","last_peered":"2023-10-09T06:14:42.679929+0000","last_clean":"2023-10-09T06:14:42.679929+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:14:42.679929+0000","last_undegraded":"2023-10-09T06:14:42.679929+0000","last_fullsized":"2023-10-09T06:14:42.679929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682509+0000","last_change":"2023-10-09T06:14:42.681947+0000","last_active":"2023-10-09T06:14:42.682509+0000","last_peered":"2023-10-09T06:14:42.682509+0000","last_clean":"2023-10-09T06:14:42.682509+0000","last_became_active":"2023-10-09T06:14:42.681620+0000","last_became_peered":"2023-10-09T06:14:42.681620+0000","last_unstale":"2023-10-09T06:14:42.682509+0000","last_undegraded":"2023-10-09T06:14:42.682509+0000","last_fullsized":"2023-10-09T06:14:42.682509+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:39.518730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686269+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:14:42.686269+0000","last_peered":"2023-10-09T06:14:42.686269+0000","last_clean":"2023-10-09T06:14:42.686269+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:14:42.686269+0000","last_undegraded":"2023-10-09T06:14:42.686269+0000","last_fullsized":"2023-10-09T06:14:42.686269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681459+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:14:42.681459+0000","last_peered":"2023-10-09T06:14:42.681459+0000","last_clean":"2023-10-09T06:14:42.681459+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:14:42.681459+0000","last_undegraded":"2023-10-09T06:14:42.681459+0000","last_fullsized":"2023-10-09T06:14:42.681459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666343+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:14:41.666343+0000","last_peered":"2023-10-09T06:14:41.666343+0000","last_clean":"2023-10-09T06:14:41.666343+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:14:41.666343+0000","last_undegraded":"2023-10-09T06:14:41.666343+0000","last_fullsized":"2023-10-09T06:14:41.666343+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430423,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302832,"kb_used_data":1640,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442320,"statfs":{"total":95995035648,"available":95684935680,"internally_reserved":0,"allocated":1679360,"data_stored":1171733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.61499999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.996},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.86699999999999999,"15min":0.85699999999999998},"min":{"1min":0.60999999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1419999999999999,"5min":1.224,"15min":1.224},"last":0.92900000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.88500000000000001,"15min":0.86499999999999999},"min":{"1min":0.71099999999999997,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.2629999999999999,"5min":1.276,"15min":1.276},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.073,"5min":0.94099999999999995,"15min":0.91900000000000004},"min":{"1min":0.83799999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.212}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92800000000000005,"15min":0.93500000000000005},"min":{"1min":0.67100000000000004,"5min":0.625,"15min":0.625},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.997},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.2689999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.0800000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:15:43 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.151,"15min":1.151},"min":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.157}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.91700000000000004,"15min":0.92100000000000004},"min":{"1min":0.68000000000000005,"5min":0.498,"15min":0.436},"max":{"1min":1.4019999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.05},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96599999999999997,"15min":0.94299999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.80400000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.0469999999999999,"15min":1.0029999999999999},"min":{"1min":0.76100000000000001,"5min":0.61099999999999999,"15min":0.48399999999999999},"max":{"1min":1.446,"5min":1.446,"15min":1.5449999999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0269999999999999,"15min":1.0600000000000001},"min":{"1min":0.51500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.377,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.1830000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.0720000000000001,"15min":1.0960000000000001},"min":{"1min":0.76900000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.5069999999999999,"5min":1.577,"15min":1.579},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.093,"15min":1.0640000000000001},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.53700000000000003},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626655,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302816,"kb_used_data":1624,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442336,"statfs":{"total":95995035648,"available":95684952064,"internally_reserved":0,"allocated":1662976,"data_stored":1170976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.90100000000000002,"15min":0.88500000000000001},"min":{"1min":0.69199999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2569999999999999,"5min":1.274,"15min":1.274},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.872,"5min":0.91400000000000003,"15min":0.93600000000000005},"min":{"1min":0.60399999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2030000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.76400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.871,"15min":0.86199999999999999},"min":{"1min":0.625,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.89300000000000002,"15min":0.89100000000000001},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":1.036}]},{"osd":2,"last update":"Mon Oct 9 06:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.97799999999999998,"15min":0.97399999999999998},"min":{"1min":0.753,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.008,"5min":0.94799999999999995,"15min":0.93799999999999994},"min":{"1min":0.751,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.98899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.071,"15min":1.071},"min":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":1.173}]},{"osd":4,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.93500000000000005,"15min":0.92200000000000004},"min":{"1min":0.74299999999999999,"5min":0.63800000000000001,"15min":0.45000000000000001},"max":{"1min":1.24,"5min":1.391,"15min":1.5289999999999999},"last":0.874},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.96799999999999997},"min":{"1min":0.71399999999999997,"5min":0.58999999999999997,"15min":0.47399999999999998},"max":{"1min":1.2789999999999999,"5min":1.3240000000000001,"15min":1.4590000000000001},"last":1.0660000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.0289999999999999,"15min":1.0089999999999999},"min":{"1min":0.89400000000000002,"5min":0.63200000000000001,"15min":0.54200000000000004},"max":{"1min":1.401,"5min":1.401,"15min":1.708},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0509999999999999,"15min":1.0369999999999999},"min":{"1min":0.72999999999999998,"5min":0.59899999999999998,"15min":0.50700000000000001},"max":{"1min":1.353,"5min":1.4019999999999999,"15min":1.4590000000000001},"last":1.232}]},{"osd":7,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.1040000000000001,"15min":1.083},"min":{"1min":0.97699999999999998,"5min":0.71899999999999997,"15min":0.47799999999999998},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.629},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.196,"5min":1.1419999999999999,"15min":1.1180000000000001},"min":{"1min":0.95599999999999996,"5min":0.72599999999999998,"15min":0.496},"max":{"1min":1.387,"5min":1.5189999999999999,"15min":1.5329999999999999},"last":1.119}]}]},{"osd":1,"up_from":93,"seq":399431958564,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":586594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":32301456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.71599999999999997,"15min":0.72199999999999998},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.4370000000000001,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.69099999999999995,"15min":0.68899999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.94999999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.67400000000000004,"15min":0.67600000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.72999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.8959999999999999,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.98699999999999999,"15min":0.97099999999999997},"min":{"1min":0.80000000000000004,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.0029999999999999,"15min":1.0089999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":1.008}]},{"osd":5,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0329999999999999,"15min":1.034},"min":{"1min":0.70699999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":2.0150000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.145,"5min":1.054,"15min":1.0389999999999999},"min":{"1min":0.82299999999999995,"5min":0.76900000000000002,"15min":0.76900000000000002},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":1.075}]},{"osd":6,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.075,"15min":1.077},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.9670000000000001,"5min":1.9670000000000001,"15min":1.9670000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.139,"15min":1.131},"min":{"1min":0.94699999999999995,"5min":0.91400000000000003,"15min":0.91400000000000003},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.98299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.1419999999999999,"15min":1.129},"min":{"1min":0.88700000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"max":{"1min":1.625,"5min":1.625,"15min":1.625},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.163,"15min":1.1759999999999999},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.88}]}]},{"osd":0,"up_from":88,"seq":377957122090,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37332,"kb_used_data":5772,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707820,"statfs":{"total":95995035648,"available":95956807680,"internally_reserved":0,"allocated":5910528,"data_stored":5371872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":32300288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61699999999999999,"15min":0.59899999999999998},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.73399999999999999,"15min":0.71599999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.113,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71299999999999997,"15min":0.71899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.89800000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.76300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1619999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.98299999999999998,"15min":0.997},"min":{"1min":0.72399999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.252,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.028}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0129999999999999,"15min":1.036},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3280000000000001,"5min":1.575,"15min":1.575},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0369999999999999,"15min":1.0569999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.462,"5min":1.472,"15min":1.472},"last":1.0609999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0580000000000001,"15min":1.077},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.532,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.0009999999999999,"15min":1.0089999999999999},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2689999999999999,"5min":1.524,"15min":1.524},"last":1.1339999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.1359999999999999,"15min":1.151},"min":{"1min":0.76500000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.458,"5min":1.667,"15min":1.667},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.097,"15min":1.113},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.325,"5min":1.7,"15min":1.7},"last":1.2789999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795038,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36664,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708488,"statfs":{"total":95995035648,"available":95957491712,"internally_reserved":0,"allocated":5292032,"data_stored":4777297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":32232382},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.65600000000000003,"15min":0.64900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.028,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.622}]},{"osd":1,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69899999999999995,"15min":0.69999999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0209999999999999,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.77500000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.96499999999999997,"15min":0.95499999999999996},"min":{"1min":0.61899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.264,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.93999999999999995,"15min":0.92700000000000005},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.008}]},{"osd":5,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.006,"15min":1.016},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.3560000000000001,"5min":1.488,"15min":1.488},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.65700000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.22,"5min":1.27,"15min":1.27},"last":0.82099999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.78800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3720000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3089999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.91200000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.091,"15min":1.095},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.389,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0940000000000001,"15min":1.0960000000000001},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.484,"5min":1.54,"15min":1.54},"last":1.032}]}]},{"osd":3,"up_from":103,"seq":442381631512,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32424,"kb_used_data":5152,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93712728,"statfs":{"total":95995035648,"available":95961833472,"internally_reserved":0,"allocated":5275648,"data_stored":4775252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":27914785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.621}]},{"osd":1,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.089,"5min":1.089,"15min":1.089},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.74199999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":1.093},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.69799999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":1.044}]},{"osd":5,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.3120000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":1.268},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.577,"5min":1.577,"15min":1.577},"last":1.117}]},{"osd":7,"last update":"Mon Oct 9 06:15:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"min":{"1min":0.752,"5min":0.752,"15min":0.752},"max":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.135},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"min":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.2110000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019121,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.96799999999999997,"15min":0.96399999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.198,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.1080000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0129999999999999,"15min":1.0249999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.96399999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.96199999999999997,"15min":0.95999999999999996},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":1.022},{"interface":"front","average":{"1min":1.006,"5min":0.98999999999999999,"15min":0.98799999999999999},"min":{"1min":0.73599999999999999,"5min":0.73299999999999998,"15min":0.73299999999999998},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.998}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.91200000000000003,"15min":0.90900000000000003},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":1.161},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.88600000000000001,"15min":0.88400000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.47,"5min":1.47,"15min":1.47},"last":0.90600000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:15:44 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.93100000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:15:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.91800000000000004,"15min":0.90000000000000002},"min":{"1min":0.65300000000000002,"5min":0.59999999999999998,"15min":0.437},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.869},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.004,"5min":0.98599999999999999,"15min":0.97099999999999997},"min":{"1min":0.58099999999999996,"5min":0.55700000000000005,"15min":0.50600000000000001},"max":{"1min":1.4390000000000001,"5min":1.466,"15min":1.9890000000000001},"last":0.85999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:15:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.008,"15min":1.0049999999999999},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.48799999999999999},"max":{"1min":1.2929999999999999,"5min":1.6180000000000001,"15min":2.0859999999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.006,"15min":0.98999999999999999},"min":{"1min":0.76200000000000001,"5min":0.53000000000000003,"15min":0.436},"max":{"1min":1.367,"5min":1.411,"15min":1.7290000000000001},"last":0.83099999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.0860000000000001,"15min":1.0649999999999999},"min":{"1min":0.92600000000000005,"5min":0.57199999999999995,"15min":0.52200000000000002},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":2.056},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0529999999999999,"15min":1.0609999999999999},"min":{"1min":0.79700000000000004,"5min":0.60499999999999998,"15min":0.504},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":2.024},"last":0.78700000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822888,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298132,"kb_used_data":1036,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447020,"statfs":{"total":95995035648,"available":95689748480,"internally_reserved":0,"allocated":1060864,"data_stored":576428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.93200000000000005,"15min":0.94499999999999995},"min":{"1min":0.57499999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.89400000000000002,"15min":0.89200000000000002},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1830000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.97599999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94199999999999995,"15min":0.94399999999999995},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.90800000000000003,"15min":0.90500000000000003},"min":{"1min":0.70099999999999996,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.94799999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.93899999999999995,"15min":0.92300000000000004},"min":{"1min":0.72599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.024,"15min":1.026},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":0.85699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":1.018},{"interface":"front","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.1559999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.90100000000000002,"15min":0.90900000000000003},"min":{"1min":0.64900000000000002,"5min":0.52600000000000002,"15min":0.42499999999999999},"max":{"1min":1.1459999999999999,"5min":1.399,"15min":1.526},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.996,"15min":0.98699999999999999},"min":{"1min":0.68300000000000005,"5min":0.55800000000000005,"15min":0.49299999999999999},"max":{"1min":1.387,"5min":1.4059999999999999,"15min":1.845},"last":1.0960000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.97399999999999998,"15min":0.99099999999999999},"min":{"1min":0.67400000000000004,"5min":0.54300000000000004,"15min":0.51200000000000001},"max":{"1min":1.2330000000000001,"5min":1.3460000000000001,"15min":1.4830000000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.1060000000000001,"15min":1.071},"min":{"1min":0.76600000000000001,"5min":0.67000000000000004,"15min":0.51400000000000001},"max":{"1min":1.4730000000000001,"5min":1.591,"15min":1.9019999999999999},"last":1.3120000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.133,"15min":1.0960000000000001},"min":{"1min":0.94299999999999995,"5min":0.63700000000000001,"15min":0.52400000000000002},"max":{"1min":1.5860000000000001,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.2609999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1399999999999999,"15min":1.125},"min":{"1min":0.95299999999999996,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":2.3969999999999998,"15min":2.3969999999999998},"last":1.246}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:16:35.886 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:16:35.886 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T06:16:35.886 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T06:16:35.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T06:16:36.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:35 smithi125 bash[77299]: cluster 2023-10-09T06:16:34.625633+0000 mgr.y (mgr.25723) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:36.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:35 smithi125 bash[77299]: audit 2023-10-09T06:16:35.016707+0000 mgr.y (mgr.25723) 336 : audit 0 from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:36.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:35 smithi125 bash[80002]: cluster 2023-10-09T06:16:34.625633+0000 mgr.y (mgr.25723) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:36.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:35 smithi125 bash[80002]: audit 2023-10-09T06:16:35.016707+0000 mgr.y (mgr.25723) 336 : audit 0 from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:35 smithi155 bash[59495]: cluster 2023-10-09T06:16:34.625633+0000 mgr.y (mgr.25723) 335 : cluster 0 pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:35 smithi155 bash[59495]: audit 2023-10-09T06:16:35.016707+0000 mgr.y (mgr.25723) 336 : audit 0 from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:38.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:37 smithi125 bash[77299]: cluster 2023-10-09T06:16:36.626468+0000 mgr.y (mgr.25723) 337 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:38.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:37 smithi125 bash[80002]: cluster 2023-10-09T06:16:36.626468+0000 mgr.y (mgr.25723) 337 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:38.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:37 smithi155 bash[59495]: cluster 2023-10-09T06:16:36.626468+0000 mgr.y (mgr.25723) 337 : cluster 0 pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:39 smithi125 bash[77299]: cluster 2023-10-09T06:16:38.627257+0000 mgr.y (mgr.25723) 338 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:39 smithi125 bash[80002]: cluster 2023-10-09T06:16:38.627257+0000 mgr.y (mgr.25723) 338 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:39.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:39 smithi155 bash[59495]: cluster 2023-10-09T06:16:38.627257+0000 mgr.y (mgr.25723) 338 : cluster 0 pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:40.696 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:41.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:41] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.33.4" 2023-10-09T06:16:42.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:41 smithi125 bash[77299]: cluster 2023-10-09T06:16:40.628145+0000 mgr.y (mgr.25723) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:42.003 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:41 smithi125 bash[80002]: cluster 2023-10-09T06:16:40.628145+0000 mgr.y (mgr.25723) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:41 smithi155 bash[59495]: cluster 2023-10-09T06:16:40.628145+0000 mgr.y (mgr.25723) 339 : cluster 0 pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:44.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:43 smithi125 bash[77299]: cluster 2023-10-09T06:16:42.628858+0000 mgr.y (mgr.25723) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:44.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:43 smithi125 bash[80002]: cluster 2023-10-09T06:16:42.628858+0000 mgr.y (mgr.25723) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:43 smithi155 bash[59495]: cluster 2023-10-09T06:16:42.628858+0000 mgr.y (mgr.25723) 340 : cluster 0 pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:16:44.283 INFO:teuthology.orchestra.run.smithi125.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T06:16:44.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:44 smithi125 bash[77299]: audit 2023-10-09T06:16:44.278612+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.125:0/2726844916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:16:44.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:44 smithi125 bash[80002]: audit 2023-10-09T06:16:44.278612+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.125:0/2726844916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:16:45.143 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T06:16:45.143 INFO:teuthology.run_tasks:Running task print... 2023-10-09T06:16:45.153 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T06:16:45.153 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T06:16:45.176 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T06:16:45.176 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T06:16:45.176 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T06:16:45.178 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T06:16:45.178 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T06:16:45.178 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T06:16:45.178 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T06:16:45.179 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:16:45.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:16:45.180 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:44 smithi155 bash[59495]: audit 2023-10-09T06:16:44.278612+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.125:0/2726844916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:16:46.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:45 smithi125 bash[77299]: cluster 2023-10-09T06:16:44.629608+0000 mgr.y (mgr.25723) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:46.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:45 smithi125 bash[80002]: cluster 2023-10-09T06:16:44.629608+0000 mgr.y (mgr.25723) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:46.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:45 smithi155 bash[59495]: cluster 2023-10-09T06:16:44.629608+0000 mgr.y (mgr.25723) 341 : cluster 0 pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:48.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:47 smithi125 bash[77299]: cluster 2023-10-09T06:16:46.630402+0000 mgr.y (mgr.25723) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:48.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:47 smithi125 bash[80002]: cluster 2023-10-09T06:16:46.630402+0000 mgr.y (mgr.25723) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:47 smithi155 bash[59495]: cluster 2023-10-09T06:16:46.630402+0000 mgr.y (mgr.25723) 342 : cluster 0 pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:49 smithi125 bash[77299]: audit 2023-10-09T06:16:48.560375+0000 mon.a (mon.0) 473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:49.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:49 smithi125 bash[77299]: cluster 2023-10-09T06:16:48.631053+0000 mgr.y (mgr.25723) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:49 smithi125 bash[80002]: audit 2023-10-09T06:16:48.560375+0000 mon.a (mon.0) 473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:49 smithi125 bash[80002]: cluster 2023-10-09T06:16:48.631053+0000 mgr.y (mgr.25723) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:49.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:49 smithi155 bash[59495]: audit 2023-10-09T06:16:48.560375+0000 mon.a (mon.0) 473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:49.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:49 smithi155 bash[59495]: cluster 2023-10-09T06:16:48.631053+0000 mgr.y (mgr.25723) 343 : cluster 0 pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:49.958 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:51.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:16:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:16:51] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T06:16:51.954 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:51 smithi125 bash[80002]: cluster 2023-10-09T06:16:50.631934+0000 mgr.y (mgr.25723) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:51.955 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:51 smithi125 bash[77299]: cluster 2023-10-09T06:16:50.631934+0000 mgr.y (mgr.25723) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:52.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:51 smithi155 bash[59495]: cluster 2023-10-09T06:16:50.631934+0000 mgr.y (mgr.25723) 344 : cluster 0 pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:53.350 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:16:53.352 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:16:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:53 smithi125 bash[80002]: cluster 2023-10-09T06:16:52.632587+0000 mgr.y (mgr.25723) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:53.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:53 smithi125 bash[77299]: cluster 2023-10-09T06:16:52.632587+0000 mgr.y (mgr.25723) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:53 smithi155 bash[59495]: cluster 2023-10-09T06:16:52.632587+0000 mgr.y (mgr.25723) 345 : cluster 0 pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:54.229 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2023-10-09T06:16:52.632186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1340996,"kb_used_data":22532,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620220,"statfs":{"total":767960285184,"available":766587105280,"internally_reserved":0,"allocated":23072768,"data_stored":19011447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1350009967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004609"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664027+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:14:41.664027+0000","last_peered":"2023-10-09T06:14:41.664027+0000","last_clean":"2023-10-09T06:14:41.664027+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:14:41.664027+0000","last_undegraded":"2023-10-09T06:14:41.664027+0000","last_fullsized":"2023-10-09T06:14:41.664027+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.666619+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:14:42.666619+0000","last_peered":"2023-10-09T06:14:42.666619+0000","last_clean":"2023-10-09T06:14:42.666619+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:14:42.666619+0000","last_undegraded":"2023-10-09T06:14:42.666619+0000","last_fullsized":"2023-10-09T06:14:42.666619+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680070+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:14:42.680070+0000","last_peered":"2023-10-09T06:14:42.680070+0000","last_clean":"2023-10-09T06:14:42.680070+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:14:42.680070+0000","last_undegraded":"2023-10-09T06:14:42.680070+0000","last_fullsized":"2023-10-09T06:14:42.680070+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689080+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:14:42.689080+0000","last_peered":"2023-10-09T06:14:42.689080+0000","last_clean":"2023-10-09T06:14:42.689080+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:14:42.689080+0000","last_undegraded":"2023-10-09T06:14:42.689080+0000","last_fullsized":"2023-10-09T06:14:42.689080+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.687434+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:14:42.687434+0000","last_peered":"2023-10-09T06:14:42.687434+0000","last_clean":"2023-10-09T06:14:42.687434+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:14:42.687434+0000","last_undegraded":"2023-10-09T06:14:42.687434+0000","last_fullsized":"2023-10-09T06:14:42.687434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.668343+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:14:41.668343+0000","last_peered":"2023-10-09T06:14:41.668343+0000","last_clean":"2023-10-09T06:14:41.668343+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:14:41.668343+0000","last_undegraded":"2023-10-09T06:14:41.668343+0000","last_fullsized":"2023-10-09T06:14:41.668343+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688026+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:14:42.688026+0000","last_peered":"2023-10-09T06:14:42.688026+0000","last_clean":"2023-10-09T06:14:42.688026+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:14:42.688026+0000","last_undegraded":"2023-10-09T06:14:42.688026+0000","last_fullsized":"2023-10-09T06:14:42.688026+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667440+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:14:41.667440+0000","last_peered":"2023-10-09T06:14:41.667440+0000","last_clean":"2023-10-09T06:14:41.667440+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:14:41.667440+0000","last_undegraded":"2023-10-09T06:14:41.667440+0000","last_fullsized":"2023-10-09T06:14:41.667440+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683126+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:14:42.683126+0000","last_peered":"2023-10-09T06:14:42.683126+0000","last_clean":"2023-10-09T06:14:42.683126+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:14:42.683126+0000","last_undegraded":"2023-10-09T06:14:42.683126+0000","last_fullsized":"2023-10-09T06:14:42.683126+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668373+0000","last_change":"2023-10-09T06:13:14.649836+0000","last_active":"2023-10-09T06:14:42.668373+0000","last_peered":"2023-10-09T06:14:42.668373+0000","last_clean":"2023-10-09T06:14:42.668373+0000","last_became_active":"2023-10-09T06:13:14.649052+0000","last_became_peered":"2023-10-09T06:13:14.649052+0000","last_unstale":"2023-10-09T06:14:42.668373+0000","last_undegraded":"2023-10-09T06:14:42.668373+0000","last_fullsized":"2023-10-09T06:14:42.668373+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:00.038681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667778+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:14:41.667778+0000","last_peered":"2023-10-09T06:14:41.667778+0000","last_clean":"2023-10-09T06:14:41.667778+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:14:41.667778+0000","last_undegraded":"2023-10-09T06:14:41.667778+0000","last_fullsized":"2023-10-09T06:14:41.667778+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668192+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:14:42.668192+0000","last_peered":"2023-10-09T06:14:42.668192+0000","last_clean":"2023-10-09T06:14:42.668192+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:14:42.668192+0000","last_undegraded":"2023-10-09T06:14:42.668192+0000","last_fullsized":"2023-10-09T06:14:42.668192+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":722,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661488+0000","last_change":"2023-10-09T06:14:12.896068+0000","last_active":"2023-10-09T06:14:41.661488+0000","last_peered":"2023-10-09T06:14:41.661488+0000","last_clean":"2023-10-09T06:14:41.661488+0000","last_became_active":"2023-10-09T06:14:12.895488+0000","last_became_peered":"2023-10-09T06:14:12.895488+0000","last_unstale":"2023-10-09T06:14:41.661488+0000","last_undegraded":"2023-10-09T06:14:41.661488+0000","last_fullsized":"2023-10-09T06:14:41.661488+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.682247+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.681892+0000","last_became_peered":"2023-10-09T06:14:42.681892+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:34.655393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":405,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675316+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:14:42.675316+0000","last_peered":"2023-10-09T06:14:42.675316+0000","last_clean":"2023-10-09T06:14:42.675316+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:14:42.675316+0000","last_undegraded":"2023-10-09T06:14:42.675316+0000","last_fullsized":"2023-10-09T06:14:42.675316+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686978+0000","last_change":"2023-10-09T06:14:42.684347+0000","last_active":"2023-10-09T06:14:42.686978+0000","last_peered":"2023-10-09T06:14:42.686978+0000","last_clean":"2023-10-09T06:14:42.686978+0000","last_became_active":"2023-10-09T06:14:42.683791+0000","last_became_peered":"2023-10-09T06:14:42.683791+0000","last_unstale":"2023-10-09T06:14:42.686978+0000","last_undegraded":"2023-10-09T06:14:42.686978+0000","last_fullsized":"2023-10-09T06:14:42.686978+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:46.220918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668111+0000","last_change":"2023-10-09T06:13:14.649857+0000","last_active":"2023-10-09T06:14:42.668111+0000","last_peered":"2023-10-09T06:14:42.668111+0000","last_clean":"2023-10-09T06:14:42.668111+0000","last_became_active":"2023-10-09T06:13:14.649499+0000","last_became_peered":"2023-10-09T06:13:14.649499+0000","last_unstale":"2023-10-09T06:14:42.668111+0000","last_undegraded":"2023-10-09T06:14:42.668111+0000","last_fullsized":"2023-10-09T06:14:42.668111+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:01.624443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676962+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:14:42.676962+0000","last_peered":"2023-10-09T06:14:42.676962+0000","last_clean":"2023-10-09T06:14:42.676962+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:14:42.676962+0000","last_undegraded":"2023-10-09T06:14:42.676962+0000","last_fullsized":"2023-10-09T06:14:42.676962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":397,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672428+0000","last_change":"2023-10-09T06:13:14.650725+0000","last_active":"2023-10-09T06:14:41.672428+0000","last_peered":"2023-10-09T06:14:41.672428+0000","last_clean":"2023-10-09T06:14:41.672428+0000","last_became_active":"2023-10-09T06:13:14.649943+0000","last_became_peered":"2023-10-09T06:13:14.649943+0000","last_unstale":"2023-10-09T06:14:41.672428+0000","last_undegraded":"2023-10-09T06:14:41.672428+0000","last_fullsized":"2023-10-09T06:14:41.672428+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668292+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:14:42.668292+0000","last_peered":"2023-10-09T06:14:42.668292+0000","last_clean":"2023-10-09T06:14:42.668292+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:14:42.668292+0000","last_undegraded":"2023-10-09T06:14:42.668292+0000","last_fullsized":"2023-10-09T06:14:42.668292+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680125+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:14:42.680125+0000","last_peered":"2023-10-09T06:14:42.680125+0000","last_clean":"2023-10-09T06:14:42.680125+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:14:42.680125+0000","last_undegraded":"2023-10-09T06:14:42.680125+0000","last_fullsized":"2023-10-09T06:14:42.680125+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":417,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671389+0000","last_change":"2023-10-09T06:13:43.436803+0000","last_active":"2023-10-09T06:14:41.671389+0000","last_peered":"2023-10-09T06:14:41.671389+0000","last_clean":"2023-10-09T06:14:41.671389+0000","last_became_active":"2023-10-09T06:13:43.435952+0000","last_became_peered":"2023-10-09T06:13:43.435952+0000","last_unstale":"2023-10-09T06:14:41.671389+0000","last_undegraded":"2023-10-09T06:14:41.671389+0000","last_fullsized":"2023-10-09T06:14:41.671389+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661963+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:14:41.661963+0000","last_peered":"2023-10-09T06:14:41.661963+0000","last_clean":"2023-10-09T06:14:41.661963+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:14:41.661963+0000","last_undegraded":"2023-10-09T06:14:41.661963+0000","last_fullsized":"2023-10-09T06:14:41.661963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689093+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:14:42.689093+0000","last_peered":"2023-10-09T06:14:42.689093+0000","last_clean":"2023-10-09T06:14:42.689093+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:14:42.689093+0000","last_undegraded":"2023-10-09T06:14:42.689093+0000","last_fullsized":"2023-10-09T06:14:42.689093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":305,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671590+0000","last_change":"2023-10-09T06:13:14.652137+0000","last_active":"2023-10-09T06:14:41.671590+0000","last_peered":"2023-10-09T06:14:41.671590+0000","last_clean":"2023-10-09T06:14:41.671590+0000","last_became_active":"2023-10-09T06:13:14.650321+0000","last_became_peered":"2023-10-09T06:13:14.650321+0000","last_unstale":"2023-10-09T06:14:41.671590+0000","last_undegraded":"2023-10-09T06:14:41.671590+0000","last_fullsized":"2023-10-09T06:14:41.671590+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":370,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.665991+0000","last_change":"2023-10-09T06:14:12.894599+0000","last_active":"2023-10-09T06:14:41.665991+0000","last_peered":"2023-10-09T06:14:41.665991+0000","last_clean":"2023-10-09T06:14:41.665991+0000","last_became_active":"2023-10-09T06:14:12.894067+0000","last_became_peered":"2023-10-09T06:14:12.894067+0000","last_unstale":"2023-10-09T06:14:41.665991+0000","last_undegraded":"2023-10-09T06:14:41.665991+0000","last_fullsized":"2023-10-09T06:14:41.665991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676067+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:14:42.676067+0000","last_peered":"2023-10-09T06:14:42.676067+0000","last_clean":"2023-10-09T06:14:42.676067+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:14:42.676067+0000","last_undegraded":"2023-10-09T06:14:42.676067+0000","last_fullsized":"2023-10-09T06:14:42.676067+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675555+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:14:42.675555+0000","last_peered":"2023-10-09T06:14:42.675555+0000","last_clean":"2023-10-09T06:14:42.675555+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:14:42.675555+0000","last_undegraded":"2023-10-09T06:14:42.675555+0000","last_fullsized":"2023-10-09T06:14:42.675555+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":460,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682883+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:14:42.682883+0000","last_peered":"2023-10-09T06:14:42.682883+0000","last_clean":"2023-10-09T06:14:42.682883+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:14:42.682883+0000","last_undegraded":"2023-10-09T06:14:42.682883+0000","last_fullsized":"2023-10-09T06:14:42.682883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686512+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:14:42.686512+0000","last_peered":"2023-10-09T06:14:42.686512+0000","last_clean":"2023-10-09T06:14:42.686512+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:14:42.686512+0000","last_undegraded":"2023-10-09T06:14:42.686512+0000","last_fullsized":"2023-10-09T06:14:42.686512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669168+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:14:41.669168+0000","last_peered":"2023-10-09T06:14:41.669168+0000","last_clean":"2023-10-09T06:14:41.669168+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:14:41.669168+0000","last_undegraded":"2023-10-09T06:14:41.669168+0000","last_fullsized":"2023-10-09T06:14:41.669168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":685,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664444+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:14:41.664444+0000","last_peered":"2023-10-09T06:14:41.664444+0000","last_clean":"2023-10-09T06:14:41.664444+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:14:41.664444+0000","last_undegraded":"2023-10-09T06:14:41.664444+0000","last_fullsized":"2023-10-09T06:14:41.664444+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674979+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:14:42.674979+0000","last_peered":"2023-10-09T06:14:42.674979+0000","last_clean":"2023-10-09T06:14:42.674979+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:14:42.674979+0000","last_undegraded":"2023-10-09T06:14:42.674979+0000","last_fullsized":"2023-10-09T06:14:42.674979+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.690962+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:14:42.690962+0000","last_peered":"2023-10-09T06:14:42.690962+0000","last_clean":"2023-10-09T06:14:42.690962+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:14:42.690962+0000","last_undegraded":"2023-10-09T06:14:42.690962+0000","last_fullsized":"2023-10-09T06:14:42.690962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":859,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676911+0000","last_change":"2023-10-09T06:13:43.431318+0000","last_active":"2023-10-09T06:14:42.676911+0000","last_peered":"2023-10-09T06:14:42.676911+0000","last_clean":"2023-10-09T06:14:42.676911+0000","last_became_active":"2023-10-09T06:13:43.430651+0000","last_became_peered":"2023-10-09T06:13:43.430651+0000","last_unstale":"2023-10-09T06:14:42.676911+0000","last_undegraded":"2023-10-09T06:14:42.676911+0000","last_fullsized":"2023-10-09T06:14:42.676911+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.401600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.685730+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:14:42.685730+0000","last_peered":"2023-10-09T06:14:42.685730+0000","last_clean":"2023-10-09T06:14:42.685730+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:14:42.685730+0000","last_undegraded":"2023-10-09T06:14:42.685730+0000","last_fullsized":"2023-10-09T06:14:42.685730+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684329+0000","last_change":"2023-10-09T06:14:42.681524+0000","last_active":"2023-10-09T06:14:42.684329+0000","last_peered":"2023-10-09T06:14:42.684329+0000","last_clean":"2023-10-09T06:14:42.684329+0000","last_became_active":"2023-10-09T06:14:42.681183+0000","last_became_peered":"2023-10-09T06:14:42.681183+0000","last_unstale":"2023-10-09T06:14:42.684329+0000","last_undegraded":"2023-10-09T06:14:42.684329+0000","last_fullsized":"2023-10-09T06:14:42.684329+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:37.001075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666609+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:14:41.666609+0000","last_peered":"2023-10-09T06:14:41.666609+0000","last_clean":"2023-10-09T06:14:41.666609+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:14:41.666609+0000","last_undegraded":"2023-10-09T06:14:41.666609+0000","last_fullsized":"2023-10-09T06:14:41.666609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667896+0000","last_change":"2023-10-09T06:14:12.892112+0000","last_active":"2023-10-09T06:14:41.667896+0000","last_peered":"2023-10-09T06:14:41.667896+0000","last_clean":"2023-10-09T06:14:41.667896+0000","last_became_active":"2023-10-09T06:14:12.891310+0000","last_became_peered":"2023-10-09T06:14:12.891310+0000","last_unstale":"2023-10-09T06:14:41.667896+0000","last_undegraded":"2023-10-09T06:14:41.667896+0000","last_fullsized":"2023-10-09T06:14:41.667896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666855+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:14:41.666855+0000","last_peered":"2023-10-09T06:14:41.666855+0000","last_clean":"2023-10-09T06:14:41.666855+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:14:41.666855+0000","last_undegraded":"2023-10-09T06:14:41.666855+0000","last_fullsized":"2023-10-09T06:14:41.666855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.662347+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:14:41.662347+0000","last_peered":"2023-10-09T06:14:41.662347+0000","last_clean":"2023-10-09T06:14:41.662347+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:14:41.662347+0000","last_undegraded":"2023-10-09T06:14:41.662347+0000","last_fullsized":"2023-10-09T06:14:41.662347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689434+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:14:42.689434+0000","last_peered":"2023-10-09T06:14:42.689434+0000","last_clean":"2023-10-09T06:14:42.689434+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:14:42.689434+0000","last_undegraded":"2023-10-09T06:14:42.689434+0000","last_fullsized":"2023-10-09T06:14:42.689434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680748+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:14:42.680748+0000","last_peered":"2023-10-09T06:14:42.680748+0000","last_clean":"2023-10-09T06:14:42.680748+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:14:42.680748+0000","last_undegraded":"2023-10-09T06:14:42.680748+0000","last_fullsized":"2023-10-09T06:14:42.680748+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668252+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:14:42.668252+0000","last_peered":"2023-10-09T06:14:42.668252+0000","last_clean":"2023-10-09T06:14:42.668252+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:14:42.668252+0000","last_undegraded":"2023-10-09T06:14:42.668252+0000","last_fullsized":"2023-10-09T06:14:42.668252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691189+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:14:42.691189+0000","last_peered":"2023-10-09T06:14:42.691189+0000","last_clean":"2023-10-09T06:14:42.691189+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:14:42.691189+0000","last_undegraded":"2023-10-09T06:14:42.691189+0000","last_fullsized":"2023-10-09T06:14:42.691189+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661806+0000","last_change":"2023-10-09T06:13:43.443807+0000","last_active":"2023-10-09T06:14:41.661806+0000","last_peered":"2023-10-09T06:14:41.661806+0000","last_clean":"2023-10-09T06:14:41.661806+0000","last_became_active":"2023-10-09T06:13:43.443290+0000","last_became_peered":"2023-10-09T06:13:43.443290+0000","last_unstale":"2023-10-09T06:14:41.661806+0000","last_undegraded":"2023-10-09T06:14:41.661806+0000","last_fullsized":"2023-10-09T06:14:41.661806+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672583+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:14:41.672583+0000","last_peered":"2023-10-09T06:14:41.672583+0000","last_clean":"2023-10-09T06:14:41.672583+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:14:41.672583+0000","last_undegraded":"2023-10-09T06:14:41.672583+0000","last_fullsized":"2023-10-09T06:14:41.672583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688520+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:14:42.688520+0000","last_peered":"2023-10-09T06:14:42.688520+0000","last_clean":"2023-10-09T06:14:42.688520+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:14:42.688520+0000","last_undegraded":"2023-10-09T06:14:42.688520+0000","last_fullsized":"2023-10-09T06:14:42.688520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672036+0000","last_change":"2023-10-09T06:13:43.436130+0000","last_active":"2023-10-09T06:14:41.672036+0000","last_peered":"2023-10-09T06:14:41.672036+0000","last_clean":"2023-10-09T06:14:41.672036+0000","last_became_active":"2023-10-09T06:13:43.435753+0000","last_became_peered":"2023-10-09T06:13:43.435753+0000","last_unstale":"2023-10-09T06:14:41.672036+0000","last_undegraded":"2023-10-09T06:14:41.672036+0000","last_fullsized":"2023-10-09T06:14:41.672036+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664290+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:14:41.664290+0000","last_peered":"2023-10-09T06:14:41.664290+0000","last_clean":"2023-10-09T06:14:41.664290+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:14:41.664290+0000","last_undegraded":"2023-10-09T06:14:41.664290+0000","last_fullsized":"2023-10-09T06:14:41.664290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671979+0000","last_change":"2023-10-09T06:13:14.651888+0000","last_active":"2023-10-09T06:14:41.671979+0000","last_peered":"2023-10-09T06:14:41.671979+0000","last_clean":"2023-10-09T06:14:41.671979+0000","last_became_active":"2023-10-09T06:13:14.651493+0000","last_became_peered":"2023-10-09T06:13:14.651493+0000","last_unstale":"2023-10-09T06:14:41.671979+0000","last_undegraded":"2023-10-09T06:14:41.671979+0000","last_fullsized":"2023-10-09T06:14:41.671979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.684353+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.683803+0000","last_became_peered":"2023-10-09T06:14:42.683803+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:55.893179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661070+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:14:41.661070+0000","last_peered":"2023-10-09T06:14:41.661070+0000","last_clean":"2023-10-09T06:14:41.661070+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:14:41.661070+0000","last_undegraded":"2023-10-09T06:14:41.661070+0000","last_fullsized":"2023-10-09T06:14:41.661070+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681933+0000","last_change":"2023-10-09T06:14:42.681256+0000","last_active":"2023-10-09T06:14:42.681933+0000","last_peered":"2023-10-09T06:14:42.681933+0000","last_clean":"2023-10-09T06:14:42.681933+0000","last_became_active":"2023-10-09T06:14:42.680856+0000","last_became_peered":"2023-10-09T06:14:42.680856+0000","last_unstale":"2023-10-09T06:14:42.681933+0000","last_undegraded":"2023-10-09T06:14:42.681933+0000","last_fullsized":"2023-10-09T06:14:42.681933+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:17.803358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.670656+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:14:41.670656+0000","last_peered":"2023-10-09T06:14:41.670656+0000","last_clean":"2023-10-09T06:14:41.670656+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:14:41.670656+0000","last_undegraded":"2023-10-09T06:14:41.670656+0000","last_fullsized":"2023-10-09T06:14:41.670656+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691481+0000","last_change":"2023-10-09T06:14:42.691481+0000","last_active":"2023-10-09T06:14:42.691481+0000","last_peered":"2023-10-09T06:14:42.691481+0000","last_clean":"2023-10-09T06:14:42.691481+0000","last_became_active":"2023-10-09T06:14:42.690934+0000","last_became_peered":"2023-10-09T06:14:42.690934+0000","last_unstale":"2023-10-09T06:14:42.691481+0000","last_undegraded":"2023-10-09T06:14:42.691481+0000","last_fullsized":"2023-10-09T06:14:42.691481+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683010+0000","last_change":"2023-10-09T06:14:42.682720+0000","last_active":"2023-10-09T06:14:42.683010+0000","last_peered":"2023-10-09T06:14:42.683010+0000","last_clean":"2023-10-09T06:14:42.683010+0000","last_became_active":"2023-10-09T06:14:42.682336+0000","last_became_peered":"2023-10-09T06:14:42.682336+0000","last_unstale":"2023-10-09T06:14:42.683010+0000","last_undegraded":"2023-10-09T06:14:42.683010+0000","last_fullsized":"2023-10-09T06:14:42.683010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:17.620826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683136+0000","last_change":"2023-10-09T06:14:42.683136+0000","last_active":"2023-10-09T06:14:42.683136+0000","last_peered":"2023-10-09T06:14:42.683136+0000","last_clean":"2023-10-09T06:14:42.683136+0000","last_became_active":"2023-10-09T06:14:42.682686+0000","last_became_peered":"2023-10-09T06:14:42.682686+0000","last_unstale":"2023-10-09T06:14:42.683136+0000","last_undegraded":"2023-10-09T06:14:42.683136+0000","last_fullsized":"2023-10-09T06:14:42.683136+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:09.556766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671277+0000","last_change":"2023-10-09T06:14:12.890850+0000","last_active":"2023-10-09T06:14:41.671277+0000","last_peered":"2023-10-09T06:14:41.671277+0000","last_clean":"2023-10-09T06:14:41.671277+0000","last_became_active":"2023-10-09T06:14:12.890340+0000","last_became_peered":"2023-10-09T06:14:12.890340+0000","last_unstale":"2023-10-09T06:14:41.671277+0000","last_undegraded":"2023-10-09T06:14:41.671277+0000","last_fullsized":"2023-10-09T06:14:41.671277+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669746+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:14:41.669746+0000","last_peered":"2023-10-09T06:14:41.669746+0000","last_clean":"2023-10-09T06:14:41.669746+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:14:41.669746+0000","last_undegraded":"2023-10-09T06:14:41.669746+0000","last_fullsized":"2023-10-09T06:14:41.669746+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":76,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667591+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:14:41.667591+0000","last_peered":"2023-10-09T06:14:41.667591+0000","last_clean":"2023-10-09T06:14:41.667591+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:14:41.667591+0000","last_undegraded":"2023-10-09T06:14:41.667591+0000","last_fullsized":"2023-10-09T06:14:41.667591+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674873+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:14:42.674873+0000","last_peered":"2023-10-09T06:14:42.674873+0000","last_clean":"2023-10-09T06:14:42.674873+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:14:42.674873+0000","last_undegraded":"2023-10-09T06:14:42.674873+0000","last_fullsized":"2023-10-09T06:14:42.674873+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689505+0000","last_change":"2023-10-09T06:14:42.689505+0000","last_active":"2023-10-09T06:14:42.689505+0000","last_peered":"2023-10-09T06:14:42.689505+0000","last_clean":"2023-10-09T06:14:42.689505+0000","last_became_active":"2023-10-09T06:14:42.688999+0000","last_became_peered":"2023-10-09T06:14:42.688999+0000","last_unstale":"2023-10-09T06:14:42.689505+0000","last_undegraded":"2023-10-09T06:14:42.689505+0000","last_fullsized":"2023-10-09T06:14:42.689505+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688566+0000","last_change":"2023-10-09T06:14:42.688566+0000","last_active":"2023-10-09T06:14:42.688566+0000","last_peered":"2023-10-09T06:14:42.688566+0000","last_clean":"2023-10-09T06:14:42.688566+0000","last_became_active":"2023-10-09T06:14:42.687716+0000","last_became_peered":"2023-10-09T06:14:42.687716+0000","last_unstale":"2023-10-09T06:14:42.688566+0000","last_undegraded":"2023-10-09T06:14:42.688566+0000","last_fullsized":"2023-10-09T06:14:42.688566+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:16.554385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664423+0000","last_change":"2023-10-09T06:13:43.433440+0000","last_active":"2023-10-09T06:14:41.664423+0000","last_peered":"2023-10-09T06:14:41.664423+0000","last_clean":"2023-10-09T06:14:41.664423+0000","last_became_active":"2023-10-09T06:13:43.432996+0000","last_became_peered":"2023-10-09T06:13:43.432996+0000","last_unstale":"2023-10-09T06:14:41.664423+0000","last_undegraded":"2023-10-09T06:14:41.664423+0000","last_fullsized":"2023-10-09T06:14:41.664423+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688101+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:14:42.688101+0000","last_peered":"2023-10-09T06:14:42.688101+0000","last_clean":"2023-10-09T06:14:42.688101+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:14:42.688101+0000","last_undegraded":"2023-10-09T06:14:42.688101+0000","last_fullsized":"2023-10-09T06:14:42.688101+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684738+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:14:42.684738+0000","last_peered":"2023-10-09T06:14:42.684738+0000","last_clean":"2023-10-09T06:14:42.684738+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:14:42.684738+0000","last_undegraded":"2023-10-09T06:14:42.684738+0000","last_fullsized":"2023-10-09T06:14:42.684738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661265+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:14:41.661265+0000","last_peered":"2023-10-09T06:14:41.661265+0000","last_clean":"2023-10-09T06:14:41.661265+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:14:41.661265+0000","last_undegraded":"2023-10-09T06:14:41.661265+0000","last_fullsized":"2023-10-09T06:14:41.661265+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.679929+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:14:42.679929+0000","last_peered":"2023-10-09T06:14:42.679929+0000","last_clean":"2023-10-09T06:14:42.679929+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:14:42.679929+0000","last_undegraded":"2023-10-09T06:14:42.679929+0000","last_fullsized":"2023-10-09T06:14:42.679929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682509+0000","last_change":"2023-10-09T06:14:42.681947+0000","last_active":"2023-10-09T06:14:42.682509+0000","last_peered":"2023-10-09T06:14:42.682509+0000","last_clean":"2023-10-09T06:14:42.682509+0000","last_became_active":"2023-10-09T06:14:42.681620+0000","last_became_peered":"2023-10-09T06:14:42.681620+0000","last_unstale":"2023-10-09T06:14:42.682509+0000","last_undegraded":"2023-10-09T06:14:42.682509+0000","last_fullsized":"2023-10-09T06:14:42.682509+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:39.518730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686269+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:14:42.686269+0000","last_peered":"2023-10-09T06:14:42.686269+0000","last_clean":"2023-10-09T06:14:42.686269+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:14:42.686269+0000","last_undegraded":"2023-10-09T06:14:42.686269+0000","last_fullsized":"2023-10-09T06:14:42.686269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681459+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:14:42.681459+0000","last_peered":"2023-10-09T06:14:42.681459+0000","last_clean":"2023-10-09T06:14:42.681459+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:14:42.681459+0000","last_undegraded":"2023-10-09T06:14:42.681459+0000","last_fullsized":"2023-10-09T06:14:42.681459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666343+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:14:41.666343+0000","last_peered":"2023-10-09T06:14:41.666343+0000","last_clean":"2023-10-09T06:14:41.666343+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:14:41.666343+0000","last_undegraded":"2023-10-09T06:14:41.666343+0000","last_fullsized":"2023-10-09T06:14:41.666343+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430426,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302832,"kb_used_data":1640,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442320,"statfs":{"total":95995035648,"available":95684935680,"internally_reserved":0,"allocated":1679360,"data_stored":1171733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.61499999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.86699999999999999,"15min":0.85699999999999998},"min":{"1min":0.60999999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1419999999999999,"5min":1.224,"15min":1.224},"last":0.85899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.88500000000000001,"15min":0.86499999999999999},"min":{"1min":0.71099999999999997,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.2629999999999999,"5min":1.276,"15min":1.276},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.073,"5min":0.94099999999999995,"15min":0.91900000000000004},"min":{"1min":0.83799999999999997,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.73499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92800000000000005,"15min":0.93500000000000005},"min":{"1min":0.67100000000000004,"5min":0.625,"15min":0.625},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.997},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.2689999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.92000000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0089999999999999,"15min":1.0149999999999999},"min":{"1min":0.58899999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.254,"5min":1.292,"15min":1.292},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.091,"5min":1.139,"15min":1.147},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.1120000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.91700000000000004,"15min":0.92100000000000004},"min":{"1min":0.68000000000000005,"5min":0.498,"15min":0.436},"max":{"1min":1.4019999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96599999999999997,"15min":0.94299999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.89000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.0469999999999999,"15min":1.0029999999999999},"min":{"1min":0.76100000000000001,"5min":0.61099999999999999,"15min":0.48399999999999999},"max":{"1min":1.446,"5min":1.446,"15min":1.5449999999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0269999999999999,"15min":1.0600000000000001},"min":{"1min":0.51500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.377,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.022}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.0720000000000001,"15min":1.0960000000000001},"min":{"1min":0.76900000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.5069999999999999,"5min":1.577,"15min":1.579},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.093,"15min":1.0640000000000001},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.53700000000000003},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.163}]}]},{"osd":6,"up_from":42,"seq":180388626658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302816,"kb_used_data":1624,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442336,"statfs":{"total":95995035648,"available":95684952064,"internally_reserved":0,"allocated":1662976,"data_stored":1170976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.90100000000000002,"15min":0.88500000000000001},"min":{"1min":0.69199999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2569999999999999,"5min":1.274,"15min":1.274},"last":1.1319999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.91400000000000003,"15min":0.93600000000000005},"min":{"1min":0.60399999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2030000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.77700000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.871,"15min":0.86199999999999999},"min":{"1min":0.625,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.89300000000000002,"15min":0.89100000000000001},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":0.99099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.97799999999999998,"15min":0.97399999999999998},"min":{"1min":0.753,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.056},{"interface":"front","average":{"1min":1.008,"5min":0.94799999999999995,"15min":0.93799999999999994},"min":{"1min":0.751,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.877}]},{"osd":3,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.05,"15min":1.0640000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.204,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.998,"5min":1.01,"15min":1.012},"min":{"1min":0.76300000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.179,"5min":1.272,"15min":1.272},"last":1.179}]},{"osd":4,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.93500000000000005,"15min":0.92200000000000004},"min":{"1min":0.74299999999999999,"5min":0.63800000000000001,"15min":0.45000000000000001},"max":{"1min":1.24,"5min":1.391,"15min":1.5289999999999999},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.96799999999999997},"min":{"1min":0.71399999999999997,"5min":0.58999999999999997,"15min":0.47399999999999998},"max":{"1min":1.2789999999999999,"5min":1.3240000000000001,"15min":1.4590000000000001},"last":1.2849999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:15:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.0289999999999999,"15min":1.0089999999999999},"min":{"1min":0.89400000000000002,"5min":0.63200000000000001,"15min":0.54200000000000004},"max":{"1min":1.401,"5min":1.401,"15min":1.708},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0509999999999999,"15min":1.0369999999999999},"min":{"1min":0.72999999999999998,"5min":0.59899999999999998,"15min":0.50700000000000001},"max":{"1min":1.353,"5min":1.4019999999999999,"15min":1.4590000000000001},"last":1.357}]},{"osd":7,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.1040000000000001,"15min":1.083},"min":{"1min":0.97699999999999998,"5min":0.71899999999999997,"15min":0.47799999999999998},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.629},"last":1.115},{"interface":"front","average":{"1min":1.196,"5min":1.1419999999999999,"15min":1.1180000000000001},"min":{"1min":0.95599999999999996,"5min":0.72599999999999998,"15min":0.496},"max":{"1min":1.387,"5min":1.5189999999999999,"15min":1.5329999999999999},"last":1.2310000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958568,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":586594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":32301456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.69999999999999996,"15min":0.71599999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.1419999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68000000000000005,"15min":0.68600000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.98499999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.67400000000000004,"15min":0.67600000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.875},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.70499999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.82499999999999996,"15min":0.83299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":1.294},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.75800000000000001,"15min":0.77400000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0620000000000001,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":1.075}]},{"osd":4,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.996,"15min":0.97399999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.3620000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.98699999999999999,"15min":1.004},"min":{"1min":0.65200000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.464,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.92200000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.02,"15min":1.03},"min":{"1min":0.75800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.7390000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":1.131},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0620000000000001,"15min":1.042},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.494,"5min":1.589,"15min":1.589},"last":0.96599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0589999999999999,"15min":1.071},"min":{"1min":0.73599999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.603,"5min":1.9670000000000001,"15min":1.9670000000000001},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.1220000000000001,"15min":1.125},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.877,"5min":1.877,"15min":1.877},"last":1.1040000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.133,"15min":1.1259999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.786,"5min":1.786,"15min":1.786},"last":1.212},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.1379999999999999,"15min":1.1679999999999999},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.5580000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":1.016}]}]},{"osd":0,"up_from":88,"seq":377957122093,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37332,"kb_used_data":5772,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707820,"statfs":{"total":95995035648,"available":95956807680,"internally_reserved":0,"allocated":5910528,"data_stored":5371872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":32300288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61699999999999999,"15min":0.59899999999999998},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.73399999999999999,"15min":0.71599999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.113,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":0.65600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71299999999999997,"15min":0.71899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.70899999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.74199999999999999,"15min":0.755},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.96799999999999997,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.76500000000000001,"15min":0.77000000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.79800000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1619999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.98299999999999998,"15min":0.997},"min":{"1min":0.72399999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.252,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.2090000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0129999999999999,"15min":1.036},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3280000000000001,"5min":1.575,"15min":1.575},"last":1.266},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0369999999999999,"15min":1.0569999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.462,"5min":1.472,"15min":1.472},"last":1.2010000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0580000000000001,"15min":1.077},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.532,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.0009999999999999,"15min":1.0089999999999999},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2689999999999999,"5min":1.524,"15min":1.524},"last":0.85499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.1359999999999999,"15min":1.151},"min":{"1min":0.76500000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.458,"5min":1.667,"15min":1.667},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.097,"15min":1.113},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.325,"5min":1.7,"15min":1.7},"last":1.0620000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795042,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36664,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708488,"statfs":{"total":95995035648,"available":95957491712,"internally_reserved":0,"allocated":5292032,"data_stored":4777297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":32232382},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.65600000000000003,"15min":0.64900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.028,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.69799999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69899999999999995,"15min":0.69999999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0209999999999999,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.748},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.98799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.72999999999999998,"15min":0.72799999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.79800000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.96499999999999997,"15min":0.95499999999999996},"min":{"1min":0.61899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.264,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.93999999999999995,"15min":0.92700000000000005},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.111}]},{"osd":5,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.006,"15min":1.016},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.3560000000000001,"5min":1.488,"15min":1.488},"last":1.246},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.65700000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.22,"5min":1.27,"15min":1.27},"last":0.92900000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.78800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3720000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.014},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3089999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":1.274}]},{"osd":7,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.091,"15min":1.095},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.389,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.214},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0940000000000001,"15min":1.0960000000000001},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.484,"5min":1.54,"15min":1.54},"last":1.1719999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631516,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32424,"kb_used_data":5152,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93712728,"statfs":{"total":95995035648,"available":95961833472,"internally_reserved":0,"allocated":5275648,"data_stored":4775252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":27914785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.66900000000000004,"15min":0.66800000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64600000000000002,"15min":0.64800000000000002},"min":{"1min":0.38800000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84299999999999997,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.63200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.70399999999999996,"15min":0.70999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.98999999999999999,"5min":1.089,"15min":1.089},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.77000000000000002,"15min":0.76200000000000001},"min":{"1min":0.59399999999999997,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":1.0029999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.82199999999999995,"15min":0.81899999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.127,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.73099999999999998,"15min":0.73399999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.91800000000000004,"15min":0.92000000000000004},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2,"5min":1.28,"15min":1.28},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.0029999999999999,"15min":1},"min":{"1min":0.64900000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3100000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.84299999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0329999999999999,"15min":1.0409999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.2589999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.069,"15min":1.0800000000000001},"min":{"1min":0.77000000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.3580000000000001,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.97399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.1240000000000001,"15min":1.1200000000000001},"min":{"1min":0.80900000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.095,"15min":1.1000000000000001},"min":{"1min":0.745,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.321,"5min":1.577,"15min":1.577},"last":0.98899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0629999999999999,"15min":1.0620000000000001},"min":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.2949999999999999,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1419999999999999,"15min":1.1379999999999999},"min":{"1min":0.93000000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.2969999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019124,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.96799999999999997,"15min":0.96399999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.198,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.0840000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0129999999999999,"15min":1.0249999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.0429999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96399999999999997,"15min":0.95999999999999996},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.264},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.97899999999999998,"15min":0.98399999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1060000000000001,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.0149999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.91200000000000003,"15min":0.90900000000000003},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.88600000000000001,"15min":0.88400000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.47,"5min":1.47,"15min":1.47},"last":0.90100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.0249999999999999},"min":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.1799999999999999,"5min":1.52,"15min":1.52},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.032,"15min":1.0289999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.3640000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:15:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.91800000000000004,"15min":0.90000000000000002},"min":{"1min":0.65300000000000002,"5min":0.59999999999999998,"15min":0.437},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.869},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.004,"5min":0.98599999999999999,"15min":0.97099999999999997},"min":{"1min":0.58099999999999996,"5min":0.55700000000000005,"15min":0.50600000000000001},"max":{"1min":1.4390000000000001,"5min":1.466,"15min":1.9890000000000001},"last":1.161}]},{"osd":6,"last update":"Mon Oct 9 06:16:38 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0209999999999999,"15min":1.02},"min":{"1min":0.86599999999999999,"5min":0.61599999999999999,"15min":0.48799999999999999},"max":{"1min":1.484,"5min":1.484,"15min":2.0859999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0269999999999999,"15min":0.998},"min":{"1min":0.83099999999999996,"5min":0.53000000000000003,"15min":0.436},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.7290000000000001},"last":1.3200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.0860000000000001,"15min":1.0649999999999999},"min":{"1min":0.92600000000000005,"5min":0.57199999999999995,"15min":0.52200000000000002},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":2.056},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0529999999999999,"15min":1.0609999999999999},"min":{"1min":0.79700000000000004,"5min":0.60499999999999998,"15min":0.504},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":2.024},"last":1.2110000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822891,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298132,"kb_used_data":1036,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447020,"statfs":{"total":95995035648,"available":95689748480,"internally_reserved":0,"allocated":1060864,"data_stored":576428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.93200000000000005,"15min":0.94499999999999995},"min":{"1min":0.57499999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.89400000000000002,"15min":0.89200000000000002},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1830000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.109}]},{"osd":1,"last update":"Mon Oct 9 06:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94199999999999995,"15min":0.94399999999999995},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.90800000000000003,"15min":0.90500000000000003},"min":{"1min":0.70099999999999996,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.94299999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.93899999999999995,"15min":0.92300000000000004},"min":{"1min":0.72599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.218},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.024,"15min":1.026},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":0.85899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.018,"15min":1.0089999999999999},"min":{"1min":0.85899999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.294,"5min":1.302,"15min":1.302},"last":1.149},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0309999999999999,"15min":1.0389999999999999},"min":{"1min":0.74299999999999999,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.1799999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.1799999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:15:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.90100000000000002,"15min":0.90900000000000003},"min":{"1min":0.64900000000000002,"5min":0.52600000000000002,"15min":0.42499999999999999},"max":{"1min":1.1459999999999999,"5min":1.399,"15min":1.526},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.996,"15min":0.98699999999999999},"min":{"1min":0.68300000000000005,"5min":0.55800000000000005,"15min":0.49299999999999999},"max":{"1min":1.387,"5min":1.4059999999999999,"15min":1.845},"last":0.80500000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97699999999999998,"15min":0.98699999999999999},"min":{"1min":0.64700000000000002,"5min":0.56599999999999995,"15min":0.53100000000000003},"max":{"1min":1.091,"5min":1.3460000000000001,"15min":1.4830000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.087,"5min":1.1020000000000001,"15min":1.075},"min":{"1min":0.81899999999999995,"5min":0.73399999999999999,"15min":0.51400000000000001},"max":{"1min":1.4059999999999999,"5min":1.4730000000000001,"15min":1.9019999999999999},"last":0.90900000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.133,"15min":1.0960000000000001},"min":{"1min":0.94299999999999995,"5min":0.63700000000000001,"15min":0.52400000000000002},"max":{"1min":1.5860000000000001,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1399999999999999,"15min":1.125},"min":{"1min":0.95299999999999996,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":2.3969999999999998,"15min":2.3969999999999998},"last":1.054}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:16:54.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:16:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:54 smithi125 bash[77299]: audit 2023-10-09T06:16:53.334343+0000 mgr.y (mgr.25723) 346 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:55.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:54 smithi125 bash[80002]: audit 2023-10-09T06:16:53.334343+0000 mgr.y (mgr.25723) 346 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:55.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:54 smithi155 bash[59495]: audit 2023-10-09T06:16:53.334343+0000 mgr.y (mgr.25723) 346 : audit 0 from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:16:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:55 smithi125 bash[77299]: cluster 2023-10-09T06:16:54.633501+0000 mgr.y (mgr.25723) 347 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:55 smithi125 bash[80002]: cluster 2023-10-09T06:16:54.633501+0000 mgr.y (mgr.25723) 347 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:56.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:55 smithi155 bash[59495]: cluster 2023-10-09T06:16:54.633501+0000 mgr.y (mgr.25723) 347 : cluster 0 pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:57 smithi125 bash[77299]: cluster 2023-10-09T06:16:56.634320+0000 mgr.y (mgr.25723) 348 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:58.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:57 smithi125 bash[80002]: cluster 2023-10-09T06:16:56.634320+0000 mgr.y (mgr.25723) 348 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:57 smithi155 bash[59495]: cluster 2023-10-09T06:16:56.634320+0000 mgr.y (mgr.25723) 348 : cluster 0 pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:59.021 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:59.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:16:59 smithi125 bash[80002]: cluster 2023-10-09T06:16:58.635010+0000 mgr.y (mgr.25723) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:59.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:16:59 smithi125 bash[77299]: cluster 2023-10-09T06:16:58.635010+0000 mgr.y (mgr.25723) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:16:59.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:16:59 smithi155 bash[59495]: cluster 2023-10-09T06:16:58.635010+0000 mgr.y (mgr.25723) 349 : cluster 0 pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:00.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:00 smithi125 bash[77299]: audit 2023-10-09T06:16:59.758794+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:00 smithi125 bash[80002]: audit 2023-10-09T06:16:59.758794+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:00 smithi155 bash[59495]: audit 2023-10-09T06:16:59.758794+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:01.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:01] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T06:17:01.847 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:01 smithi155 bash[59495]: cluster 2023-10-09T06:17:00.635806+0000 mgr.y (mgr.25723) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:01.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:01 smithi125 bash[77299]: cluster 2023-10-09T06:17:00.635806+0000 mgr.y (mgr.25723) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:01 smithi125 bash[80002]: cluster 2023-10-09T06:17:00.635806+0000 mgr.y (mgr.25723) 350 : cluster 0 pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:02.479 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:17:02.482 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:17:03.393 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":284,"stamp":"2023-10-09T06:17:00.635250+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6303,"num_read_kb":13760,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1340996,"kb_used_data":22532,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620220,"statfs":{"total":767960285184,"available":766587105280,"internally_reserved":0,"allocated":23072768,"data_stored":19011447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1350009967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004573"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664027+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:14:41.664027+0000","last_peered":"2023-10-09T06:14:41.664027+0000","last_clean":"2023-10-09T06:14:41.664027+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:14:41.664027+0000","last_undegraded":"2023-10-09T06:14:41.664027+0000","last_fullsized":"2023-10-09T06:14:41.664027+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.666619+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:14:42.666619+0000","last_peered":"2023-10-09T06:14:42.666619+0000","last_clean":"2023-10-09T06:14:42.666619+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:14:42.666619+0000","last_undegraded":"2023-10-09T06:14:42.666619+0000","last_fullsized":"2023-10-09T06:14:42.666619+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680070+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:14:42.680070+0000","last_peered":"2023-10-09T06:14:42.680070+0000","last_clean":"2023-10-09T06:14:42.680070+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:14:42.680070+0000","last_undegraded":"2023-10-09T06:14:42.680070+0000","last_fullsized":"2023-10-09T06:14:42.680070+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689080+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:14:42.689080+0000","last_peered":"2023-10-09T06:14:42.689080+0000","last_clean":"2023-10-09T06:14:42.689080+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:14:42.689080+0000","last_undegraded":"2023-10-09T06:14:42.689080+0000","last_fullsized":"2023-10-09T06:14:42.689080+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.687434+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:14:42.687434+0000","last_peered":"2023-10-09T06:14:42.687434+0000","last_clean":"2023-10-09T06:14:42.687434+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:14:42.687434+0000","last_undegraded":"2023-10-09T06:14:42.687434+0000","last_fullsized":"2023-10-09T06:14:42.687434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":56,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.668343+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:14:41.668343+0000","last_peered":"2023-10-09T06:14:41.668343+0000","last_clean":"2023-10-09T06:14:41.668343+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:14:41.668343+0000","last_undegraded":"2023-10-09T06:14:41.668343+0000","last_fullsized":"2023-10-09T06:14:41.668343+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688026+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:14:42.688026+0000","last_peered":"2023-10-09T06:14:42.688026+0000","last_clean":"2023-10-09T06:14:42.688026+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:14:42.688026+0000","last_undegraded":"2023-10-09T06:14:42.688026+0000","last_fullsized":"2023-10-09T06:14:42.688026+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667440+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:14:41.667440+0000","last_peered":"2023-10-09T06:14:41.667440+0000","last_clean":"2023-10-09T06:14:41.667440+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:14:41.667440+0000","last_undegraded":"2023-10-09T06:14:41.667440+0000","last_fullsized":"2023-10-09T06:14:41.667440+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683126+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:14:42.683126+0000","last_peered":"2023-10-09T06:14:42.683126+0000","last_clean":"2023-10-09T06:14:42.683126+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:14:42.683126+0000","last_undegraded":"2023-10-09T06:14:42.683126+0000","last_fullsized":"2023-10-09T06:14:42.683126+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668373+0000","last_change":"2023-10-09T06:13:14.649836+0000","last_active":"2023-10-09T06:14:42.668373+0000","last_peered":"2023-10-09T06:14:42.668373+0000","last_clean":"2023-10-09T06:14:42.668373+0000","last_became_active":"2023-10-09T06:13:14.649052+0000","last_became_peered":"2023-10-09T06:13:14.649052+0000","last_unstale":"2023-10-09T06:14:42.668373+0000","last_undegraded":"2023-10-09T06:14:42.668373+0000","last_fullsized":"2023-10-09T06:14:42.668373+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:41:00.038681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667778+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:14:41.667778+0000","last_peered":"2023-10-09T06:14:41.667778+0000","last_clean":"2023-10-09T06:14:41.667778+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:14:41.667778+0000","last_undegraded":"2023-10-09T06:14:41.667778+0000","last_fullsized":"2023-10-09T06:14:41.667778+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668192+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:14:42.668192+0000","last_peered":"2023-10-09T06:14:42.668192+0000","last_clean":"2023-10-09T06:14:42.668192+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:14:42.668192+0000","last_undegraded":"2023-10-09T06:14:42.668192+0000","last_fullsized":"2023-10-09T06:14:42.668192+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":722,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661488+0000","last_change":"2023-10-09T06:14:12.896068+0000","last_active":"2023-10-09T06:14:41.661488+0000","last_peered":"2023-10-09T06:14:41.661488+0000","last_clean":"2023-10-09T06:14:41.661488+0000","last_became_active":"2023-10-09T06:14:12.895488+0000","last_became_peered":"2023-10-09T06:14:12.895488+0000","last_unstale":"2023-10-09T06:14:41.661488+0000","last_undegraded":"2023-10-09T06:14:41.661488+0000","last_fullsized":"2023-10-09T06:14:41.661488+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.682247+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.681892+0000","last_became_peered":"2023-10-09T06:14:42.681892+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:34.655393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":405,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675316+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:14:42.675316+0000","last_peered":"2023-10-09T06:14:42.675316+0000","last_clean":"2023-10-09T06:14:42.675316+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:14:42.675316+0000","last_undegraded":"2023-10-09T06:14:42.675316+0000","last_fullsized":"2023-10-09T06:14:42.675316+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686978+0000","last_change":"2023-10-09T06:14:42.684347+0000","last_active":"2023-10-09T06:14:42.686978+0000","last_peered":"2023-10-09T06:14:42.686978+0000","last_clean":"2023-10-09T06:14:42.686978+0000","last_became_active":"2023-10-09T06:14:42.683791+0000","last_became_peered":"2023-10-09T06:14:42.683791+0000","last_unstale":"2023-10-09T06:14:42.686978+0000","last_undegraded":"2023-10-09T06:14:42.686978+0000","last_fullsized":"2023-10-09T06:14:42.686978+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:46.220918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668111+0000","last_change":"2023-10-09T06:13:14.649857+0000","last_active":"2023-10-09T06:14:42.668111+0000","last_peered":"2023-10-09T06:14:42.668111+0000","last_clean":"2023-10-09T06:14:42.668111+0000","last_became_active":"2023-10-09T06:13:14.649499+0000","last_became_peered":"2023-10-09T06:13:14.649499+0000","last_unstale":"2023-10-09T06:14:42.668111+0000","last_undegraded":"2023-10-09T06:14:42.668111+0000","last_fullsized":"2023-10-09T06:14:42.668111+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:07:01.624443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676962+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:14:42.676962+0000","last_peered":"2023-10-09T06:14:42.676962+0000","last_clean":"2023-10-09T06:14:42.676962+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:14:42.676962+0000","last_undegraded":"2023-10-09T06:14:42.676962+0000","last_fullsized":"2023-10-09T06:14:42.676962+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":397,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672428+0000","last_change":"2023-10-09T06:13:14.650725+0000","last_active":"2023-10-09T06:14:41.672428+0000","last_peered":"2023-10-09T06:14:41.672428+0000","last_clean":"2023-10-09T06:14:41.672428+0000","last_became_active":"2023-10-09T06:13:14.649943+0000","last_became_peered":"2023-10-09T06:13:14.649943+0000","last_unstale":"2023-10-09T06:14:41.672428+0000","last_undegraded":"2023-10-09T06:14:41.672428+0000","last_fullsized":"2023-10-09T06:14:41.672428+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668292+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:14:42.668292+0000","last_peered":"2023-10-09T06:14:42.668292+0000","last_clean":"2023-10-09T06:14:42.668292+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:14:42.668292+0000","last_undegraded":"2023-10-09T06:14:42.668292+0000","last_fullsized":"2023-10-09T06:14:42.668292+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.680125+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:14:42.680125+0000","last_peered":"2023-10-09T06:14:42.680125+0000","last_clean":"2023-10-09T06:14:42.680125+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:14:42.680125+0000","last_undegraded":"2023-10-09T06:14:42.680125+0000","last_fullsized":"2023-10-09T06:14:42.680125+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":417,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671389+0000","last_change":"2023-10-09T06:13:43.436803+0000","last_active":"2023-10-09T06:14:41.671389+0000","last_peered":"2023-10-09T06:14:41.671389+0000","last_clean":"2023-10-09T06:14:41.671389+0000","last_became_active":"2023-10-09T06:13:43.435952+0000","last_became_peered":"2023-10-09T06:13:43.435952+0000","last_unstale":"2023-10-09T06:14:41.671389+0000","last_undegraded":"2023-10-09T06:14:41.671389+0000","last_fullsized":"2023-10-09T06:14:41.671389+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661963+0000","last_change":"2023-10-09T06:01:50.674809+0000","last_active":"2023-10-09T06:14:41.661963+0000","last_peered":"2023-10-09T06:14:41.661963+0000","last_clean":"2023-10-09T06:14:41.661963+0000","last_became_active":"2023-10-09T06:01:50.674490+0000","last_became_peered":"2023-10-09T06:01:50.674490+0000","last_unstale":"2023-10-09T06:14:41.661963+0000","last_undegraded":"2023-10-09T06:14:41.661963+0000","last_fullsized":"2023-10-09T06:14:41.661963+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689093+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:14:42.689093+0000","last_peered":"2023-10-09T06:14:42.689093+0000","last_clean":"2023-10-09T06:14:42.689093+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:14:42.689093+0000","last_undegraded":"2023-10-09T06:14:42.689093+0000","last_fullsized":"2023-10-09T06:14:42.689093+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":305,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671590+0000","last_change":"2023-10-09T06:13:14.652137+0000","last_active":"2023-10-09T06:14:41.671590+0000","last_peered":"2023-10-09T06:14:41.671590+0000","last_clean":"2023-10-09T06:14:41.671590+0000","last_became_active":"2023-10-09T06:13:14.650321+0000","last_became_peered":"2023-10-09T06:13:14.650321+0000","last_unstale":"2023-10-09T06:14:41.671590+0000","last_undegraded":"2023-10-09T06:14:41.671590+0000","last_fullsized":"2023-10-09T06:14:41.671590+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":370,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.665991+0000","last_change":"2023-10-09T06:14:12.894599+0000","last_active":"2023-10-09T06:14:41.665991+0000","last_peered":"2023-10-09T06:14:41.665991+0000","last_clean":"2023-10-09T06:14:41.665991+0000","last_became_active":"2023-10-09T06:14:12.894067+0000","last_became_peered":"2023-10-09T06:14:12.894067+0000","last_unstale":"2023-10-09T06:14:41.665991+0000","last_undegraded":"2023-10-09T06:14:41.665991+0000","last_fullsized":"2023-10-09T06:14:41.665991+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676067+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:14:42.676067+0000","last_peered":"2023-10-09T06:14:42.676067+0000","last_clean":"2023-10-09T06:14:42.676067+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:14:42.676067+0000","last_undegraded":"2023-10-09T06:14:42.676067+0000","last_fullsized":"2023-10-09T06:14:42.676067+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.675555+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:14:42.675555+0000","last_peered":"2023-10-09T06:14:42.675555+0000","last_clean":"2023-10-09T06:14:42.675555+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:14:42.675555+0000","last_undegraded":"2023-10-09T06:14:42.675555+0000","last_fullsized":"2023-10-09T06:14:42.675555+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":460,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682883+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:14:42.682883+0000","last_peered":"2023-10-09T06:14:42.682883+0000","last_clean":"2023-10-09T06:14:42.682883+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:14:42.682883+0000","last_undegraded":"2023-10-09T06:14:42.682883+0000","last_fullsized":"2023-10-09T06:14:42.682883+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":64,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686512+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:14:42.686512+0000","last_peered":"2023-10-09T06:14:42.686512+0000","last_clean":"2023-10-09T06:14:42.686512+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:14:42.686512+0000","last_undegraded":"2023-10-09T06:14:42.686512+0000","last_fullsized":"2023-10-09T06:14:42.686512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669168+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:14:41.669168+0000","last_peered":"2023-10-09T06:14:41.669168+0000","last_clean":"2023-10-09T06:14:41.669168+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:14:41.669168+0000","last_undegraded":"2023-10-09T06:14:41.669168+0000","last_fullsized":"2023-10-09T06:14:41.669168+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":685,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664444+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:14:41.664444+0000","last_peered":"2023-10-09T06:14:41.664444+0000","last_clean":"2023-10-09T06:14:41.664444+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:14:41.664444+0000","last_undegraded":"2023-10-09T06:14:41.664444+0000","last_fullsized":"2023-10-09T06:14:41.664444+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674979+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:14:42.674979+0000","last_peered":"2023-10-09T06:14:42.674979+0000","last_clean":"2023-10-09T06:14:42.674979+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:14:42.674979+0000","last_undegraded":"2023-10-09T06:14:42.674979+0000","last_fullsized":"2023-10-09T06:14:42.674979+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.690962+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:14:42.690962+0000","last_peered":"2023-10-09T06:14:42.690962+0000","last_clean":"2023-10-09T06:14:42.690962+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:14:42.690962+0000","last_undegraded":"2023-10-09T06:14:42.690962+0000","last_fullsized":"2023-10-09T06:14:42.690962+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":859,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.676911+0000","last_change":"2023-10-09T06:13:43.431318+0000","last_active":"2023-10-09T06:14:42.676911+0000","last_peered":"2023-10-09T06:14:42.676911+0000","last_clean":"2023-10-09T06:14:42.676911+0000","last_became_active":"2023-10-09T06:13:43.430651+0000","last_became_peered":"2023-10-09T06:13:43.430651+0000","last_unstale":"2023-10-09T06:14:42.676911+0000","last_undegraded":"2023-10-09T06:14:42.676911+0000","last_fullsized":"2023-10-09T06:14:42.676911+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.401600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.685730+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:14:42.685730+0000","last_peered":"2023-10-09T06:14:42.685730+0000","last_clean":"2023-10-09T06:14:42.685730+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:14:42.685730+0000","last_undegraded":"2023-10-09T06:14:42.685730+0000","last_fullsized":"2023-10-09T06:14:42.685730+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684329+0000","last_change":"2023-10-09T06:14:42.681524+0000","last_active":"2023-10-09T06:14:42.684329+0000","last_peered":"2023-10-09T06:14:42.684329+0000","last_clean":"2023-10-09T06:14:42.684329+0000","last_became_active":"2023-10-09T06:14:42.681183+0000","last_became_peered":"2023-10-09T06:14:42.681183+0000","last_unstale":"2023-10-09T06:14:42.684329+0000","last_undegraded":"2023-10-09T06:14:42.684329+0000","last_fullsized":"2023-10-09T06:14:42.684329+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:37.001075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666609+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:14:41.666609+0000","last_peered":"2023-10-09T06:14:41.666609+0000","last_clean":"2023-10-09T06:14:41.666609+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:14:41.666609+0000","last_undegraded":"2023-10-09T06:14:41.666609+0000","last_fullsized":"2023-10-09T06:14:41.666609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667896+0000","last_change":"2023-10-09T06:14:12.892112+0000","last_active":"2023-10-09T06:14:41.667896+0000","last_peered":"2023-10-09T06:14:41.667896+0000","last_clean":"2023-10-09T06:14:41.667896+0000","last_became_active":"2023-10-09T06:14:12.891310+0000","last_became_peered":"2023-10-09T06:14:12.891310+0000","last_unstale":"2023-10-09T06:14:41.667896+0000","last_undegraded":"2023-10-09T06:14:41.667896+0000","last_fullsized":"2023-10-09T06:14:41.667896+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666855+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:14:41.666855+0000","last_peered":"2023-10-09T06:14:41.666855+0000","last_clean":"2023-10-09T06:14:41.666855+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:14:41.666855+0000","last_undegraded":"2023-10-09T06:14:41.666855+0000","last_fullsized":"2023-10-09T06:14:41.666855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.662347+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:14:41.662347+0000","last_peered":"2023-10-09T06:14:41.662347+0000","last_clean":"2023-10-09T06:14:41.662347+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:14:41.662347+0000","last_undegraded":"2023-10-09T06:14:41.662347+0000","last_fullsized":"2023-10-09T06:14:41.662347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689434+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:14:42.689434+0000","last_peered":"2023-10-09T06:14:42.689434+0000","last_clean":"2023-10-09T06:14:42.689434+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:14:42.689434+0000","last_undegraded":"2023-10-09T06:14:42.689434+0000","last_fullsized":"2023-10-09T06:14:42.689434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683781+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:14:42.683781+0000","last_peered":"2023-10-09T06:14:42.683781+0000","last_clean":"2023-10-09T06:14:42.683781+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:14:42.683781+0000","last_undegraded":"2023-10-09T06:14:42.683781+0000","last_fullsized":"2023-10-09T06:14:42.683781+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.668252+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:14:42.668252+0000","last_peered":"2023-10-09T06:14:42.668252+0000","last_clean":"2023-10-09T06:14:42.668252+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:14:42.668252+0000","last_undegraded":"2023-10-09T06:14:42.668252+0000","last_fullsized":"2023-10-09T06:14:42.668252+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691189+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:14:42.691189+0000","last_peered":"2023-10-09T06:14:42.691189+0000","last_clean":"2023-10-09T06:14:42.691189+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:14:42.691189+0000","last_undegraded":"2023-10-09T06:14:42.691189+0000","last_fullsized":"2023-10-09T06:14:42.691189+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661806+0000","last_change":"2023-10-09T06:13:43.443807+0000","last_active":"2023-10-09T06:14:41.661806+0000","last_peered":"2023-10-09T06:14:41.661806+0000","last_clean":"2023-10-09T06:14:41.661806+0000","last_became_active":"2023-10-09T06:13:43.443290+0000","last_became_peered":"2023-10-09T06:13:43.443290+0000","last_unstale":"2023-10-09T06:14:41.661806+0000","last_undegraded":"2023-10-09T06:14:41.661806+0000","last_fullsized":"2023-10-09T06:14:41.661806+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":55,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672583+0000","last_change":"2023-10-09T06:01:50.653578+0000","last_active":"2023-10-09T06:14:41.672583+0000","last_peered":"2023-10-09T06:14:41.672583+0000","last_clean":"2023-10-09T06:14:41.672583+0000","last_became_active":"2023-10-09T06:01:50.651280+0000","last_became_peered":"2023-10-09T06:01:50.651280+0000","last_unstale":"2023-10-09T06:14:41.672583+0000","last_undegraded":"2023-10-09T06:14:41.672583+0000","last_fullsized":"2023-10-09T06:14:41.672583+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688520+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:14:42.688520+0000","last_peered":"2023-10-09T06:14:42.688520+0000","last_clean":"2023-10-09T06:14:42.688520+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:14:42.688520+0000","last_undegraded":"2023-10-09T06:14:42.688520+0000","last_fullsized":"2023-10-09T06:14:42.688520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.672036+0000","last_change":"2023-10-09T06:13:43.436130+0000","last_active":"2023-10-09T06:14:41.672036+0000","last_peered":"2023-10-09T06:14:41.672036+0000","last_clean":"2023-10-09T06:14:41.672036+0000","last_became_active":"2023-10-09T06:13:43.435753+0000","last_became_peered":"2023-10-09T06:13:43.435753+0000","last_unstale":"2023-10-09T06:14:41.672036+0000","last_undegraded":"2023-10-09T06:14:41.672036+0000","last_fullsized":"2023-10-09T06:14:41.672036+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664290+0000","last_change":"2023-10-09T06:01:50.672082+0000","last_active":"2023-10-09T06:14:41.664290+0000","last_peered":"2023-10-09T06:14:41.664290+0000","last_clean":"2023-10-09T06:14:41.664290+0000","last_became_active":"2023-10-09T06:01:50.671613+0000","last_became_peered":"2023-10-09T06:01:50.671613+0000","last_unstale":"2023-10-09T06:14:41.664290+0000","last_undegraded":"2023-10-09T06:14:41.664290+0000","last_fullsized":"2023-10-09T06:14:41.664290+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671979+0000","last_change":"2023-10-09T06:13:14.651888+0000","last_active":"2023-10-09T06:14:41.671979+0000","last_peered":"2023-10-09T06:14:41.671979+0000","last_clean":"2023-10-09T06:14:41.671979+0000","last_became_active":"2023-10-09T06:13:14.651493+0000","last_became_peered":"2023-10-09T06:13:14.651493+0000","last_unstale":"2023-10-09T06:14:41.671979+0000","last_undegraded":"2023-10-09T06:14:41.671979+0000","last_fullsized":"2023-10-09T06:14:41.671979+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684353+0000","last_change":"2023-10-09T06:14:42.684353+0000","last_active":"2023-10-09T06:14:42.684353+0000","last_peered":"2023-10-09T06:14:42.684353+0000","last_clean":"2023-10-09T06:14:42.684353+0000","last_became_active":"2023-10-09T06:14:42.683803+0000","last_became_peered":"2023-10-09T06:14:42.683803+0000","last_unstale":"2023-10-09T06:14:42.684353+0000","last_undegraded":"2023-10-09T06:14:42.684353+0000","last_fullsized":"2023-10-09T06:14:42.684353+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:55.893179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":78,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661070+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:14:41.661070+0000","last_peered":"2023-10-09T06:14:41.661070+0000","last_clean":"2023-10-09T06:14:41.661070+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:14:41.661070+0000","last_undegraded":"2023-10-09T06:14:41.661070+0000","last_fullsized":"2023-10-09T06:14:41.661070+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681933+0000","last_change":"2023-10-09T06:14:42.681256+0000","last_active":"2023-10-09T06:14:42.681933+0000","last_peered":"2023-10-09T06:14:42.681933+0000","last_clean":"2023-10-09T06:14:42.681933+0000","last_became_active":"2023-10-09T06:14:42.680856+0000","last_became_peered":"2023-10-09T06:14:42.680856+0000","last_unstale":"2023-10-09T06:14:42.681933+0000","last_undegraded":"2023-10-09T06:14:42.681933+0000","last_fullsized":"2023-10-09T06:14:42.681933+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:17.803358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.670656+0000","last_change":"2023-10-09T06:01:50.655186+0000","last_active":"2023-10-09T06:14:41.670656+0000","last_peered":"2023-10-09T06:14:41.670656+0000","last_clean":"2023-10-09T06:14:41.670656+0000","last_became_active":"2023-10-09T06:01:50.654867+0000","last_became_peered":"2023-10-09T06:01:50.654867+0000","last_unstale":"2023-10-09T06:14:41.670656+0000","last_undegraded":"2023-10-09T06:14:41.670656+0000","last_fullsized":"2023-10-09T06:14:41.670656+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.691481+0000","last_change":"2023-10-09T06:14:42.691481+0000","last_active":"2023-10-09T06:14:42.691481+0000","last_peered":"2023-10-09T06:14:42.691481+0000","last_clean":"2023-10-09T06:14:42.691481+0000","last_became_active":"2023-10-09T06:14:42.690934+0000","last_became_peered":"2023-10-09T06:14:42.690934+0000","last_unstale":"2023-10-09T06:14:42.691481+0000","last_undegraded":"2023-10-09T06:14:42.691481+0000","last_fullsized":"2023-10-09T06:14:42.691481+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683010+0000","last_change":"2023-10-09T06:14:42.682720+0000","last_active":"2023-10-09T06:14:42.683010+0000","last_peered":"2023-10-09T06:14:42.683010+0000","last_clean":"2023-10-09T06:14:42.683010+0000","last_became_active":"2023-10-09T06:14:42.682336+0000","last_became_peered":"2023-10-09T06:14:42.682336+0000","last_unstale":"2023-10-09T06:14:42.683010+0000","last_undegraded":"2023-10-09T06:14:42.683010+0000","last_fullsized":"2023-10-09T06:14:42.683010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:17.620826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.683136+0000","last_change":"2023-10-09T06:14:42.683136+0000","last_active":"2023-10-09T06:14:42.683136+0000","last_peered":"2023-10-09T06:14:42.683136+0000","last_clean":"2023-10-09T06:14:42.683136+0000","last_became_active":"2023-10-09T06:14:42.682686+0000","last_became_peered":"2023-10-09T06:14:42.682686+0000","last_unstale":"2023-10-09T06:14:42.683136+0000","last_undegraded":"2023-10-09T06:14:42.683136+0000","last_fullsized":"2023-10-09T06:14:42.683136+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:09.556766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":71,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.671277+0000","last_change":"2023-10-09T06:14:12.890850+0000","last_active":"2023-10-09T06:14:41.671277+0000","last_peered":"2023-10-09T06:14:41.671277+0000","last_clean":"2023-10-09T06:14:41.671277+0000","last_became_active":"2023-10-09T06:14:12.890340+0000","last_became_peered":"2023-10-09T06:14:12.890340+0000","last_unstale":"2023-10-09T06:14:41.671277+0000","last_undegraded":"2023-10-09T06:14:41.671277+0000","last_fullsized":"2023-10-09T06:14:41.671277+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.669746+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:14:41.669746+0000","last_peered":"2023-10-09T06:14:41.669746+0000","last_clean":"2023-10-09T06:14:41.669746+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:14:41.669746+0000","last_undegraded":"2023-10-09T06:14:41.669746+0000","last_fullsized":"2023-10-09T06:14:41.669746+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":76,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.667591+0000","last_change":"2023-10-09T06:01:50.653765+0000","last_active":"2023-10-09T06:14:41.667591+0000","last_peered":"2023-10-09T06:14:41.667591+0000","last_clean":"2023-10-09T06:14:41.667591+0000","last_became_active":"2023-10-09T06:01:50.653272+0000","last_became_peered":"2023-10-09T06:01:50.653272+0000","last_unstale":"2023-10-09T06:14:41.667591+0000","last_undegraded":"2023-10-09T06:14:41.667591+0000","last_fullsized":"2023-10-09T06:14:41.667591+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.674873+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:14:42.674873+0000","last_peered":"2023-10-09T06:14:42.674873+0000","last_clean":"2023-10-09T06:14:42.674873+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:14:42.674873+0000","last_undegraded":"2023-10-09T06:14:42.674873+0000","last_fullsized":"2023-10-09T06:14:42.674873+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.689505+0000","last_change":"2023-10-09T06:14:42.689505+0000","last_active":"2023-10-09T06:14:42.689505+0000","last_peered":"2023-10-09T06:14:42.689505+0000","last_clean":"2023-10-09T06:14:42.689505+0000","last_became_active":"2023-10-09T06:14:42.688999+0000","last_became_peered":"2023-10-09T06:14:42.688999+0000","last_unstale":"2023-10-09T06:14:42.689505+0000","last_undegraded":"2023-10-09T06:14:42.689505+0000","last_fullsized":"2023-10-09T06:14:42.689505+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688566+0000","last_change":"2023-10-09T06:14:42.688566+0000","last_active":"2023-10-09T06:14:42.688566+0000","last_peered":"2023-10-09T06:14:42.688566+0000","last_clean":"2023-10-09T06:14:42.688566+0000","last_became_active":"2023-10-09T06:14:42.687716+0000","last_became_peered":"2023-10-09T06:14:42.687716+0000","last_unstale":"2023-10-09T06:14:42.688566+0000","last_undegraded":"2023-10-09T06:14:42.688566+0000","last_fullsized":"2023-10-09T06:14:42.688566+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:16.554385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.664423+0000","last_change":"2023-10-09T06:13:43.433440+0000","last_active":"2023-10-09T06:14:41.664423+0000","last_peered":"2023-10-09T06:14:41.664423+0000","last_clean":"2023-10-09T06:14:41.664423+0000","last_became_active":"2023-10-09T06:13:43.432996+0000","last_became_peered":"2023-10-09T06:13:43.432996+0000","last_unstale":"2023-10-09T06:14:41.664423+0000","last_undegraded":"2023-10-09T06:14:41.664423+0000","last_fullsized":"2023-10-09T06:14:41.664423+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.688101+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:14:42.688101+0000","last_peered":"2023-10-09T06:14:42.688101+0000","last_clean":"2023-10-09T06:14:42.688101+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:14:42.688101+0000","last_undegraded":"2023-10-09T06:14:42.688101+0000","last_fullsized":"2023-10-09T06:14:42.688101+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.684738+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:14:42.684738+0000","last_peered":"2023-10-09T06:14:42.684738+0000","last_clean":"2023-10-09T06:14:42.684738+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:14:42.684738+0000","last_undegraded":"2023-10-09T06:14:42.684738+0000","last_fullsized":"2023-10-09T06:14:42.684738+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.661265+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:14:41.661265+0000","last_peered":"2023-10-09T06:14:41.661265+0000","last_clean":"2023-10-09T06:14:41.661265+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:14:41.661265+0000","last_undegraded":"2023-10-09T06:14:41.661265+0000","last_fullsized":"2023-10-09T06:14:41.661265+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":88,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.679929+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:14:42.679929+0000","last_peered":"2023-10-09T06:14:42.679929+0000","last_clean":"2023-10-09T06:14:42.679929+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:14:42.679929+0000","last_undegraded":"2023-10-09T06:14:42.679929+0000","last_fullsized":"2023-10-09T06:14:42.679929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.682509+0000","last_change":"2023-10-09T06:14:42.681947+0000","last_active":"2023-10-09T06:14:42.682509+0000","last_peered":"2023-10-09T06:14:42.682509+0000","last_clean":"2023-10-09T06:14:42.682509+0000","last_became_active":"2023-10-09T06:14:42.681620+0000","last_became_peered":"2023-10-09T06:14:42.681620+0000","last_unstale":"2023-10-09T06:14:42.682509+0000","last_undegraded":"2023-10-09T06:14:42.682509+0000","last_fullsized":"2023-10-09T06:14:42.682509+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:39.518730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":48,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.686269+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:14:42.686269+0000","last_peered":"2023-10-09T06:14:42.686269+0000","last_clean":"2023-10-09T06:14:42.686269+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:14:42.686269+0000","last_undegraded":"2023-10-09T06:14:42.686269+0000","last_fullsized":"2023-10-09T06:14:42.686269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-09T06:14:42.681459+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:14:42.681459+0000","last_peered":"2023-10-09T06:14:42.681459+0000","last_clean":"2023-10-09T06:14:42.681459+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:14:42.681459+0000","last_undegraded":"2023-10-09T06:14:42.681459+0000","last_fullsized":"2023-10-09T06:14:42.681459+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":103,"state":"active+clean","last_fresh":"2023-10-09T06:14:41.666343+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:14:41.666343+0000","last_peered":"2023-10-09T06:14:41.666343+0000","last_clean":"2023-10-09T06:14:41.666343+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:14:41.666343+0000","last_undegraded":"2023-10-09T06:14:41.666343+0000","last_fullsized":"2023-10-09T06:14:41.666343+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430428,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302832,"kb_used_data":1640,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442320,"statfs":{"total":95995035648,"available":95684935680,"internally_reserved":0,"allocated":1679360,"data_stored":1171733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.873,"15min":0.86499999999999999},"min":{"1min":0.61499999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.325},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.86699999999999999,"15min":0.85699999999999998},"min":{"1min":0.60999999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1419999999999999,"5min":1.224,"15min":1.224},"last":1.198}]},{"osd":1,"last update":"Mon Oct 9 06:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.90700000000000003,"15min":0.872},"min":{"1min":0.73499999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.94699999999999995,"15min":0.92100000000000004},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.212,"5min":1.319,"15min":1.319},"last":0.99299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92800000000000005,"15min":0.93500000000000005},"min":{"1min":0.67100000000000004,"5min":0.625,"15min":0.625},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.04,"5min":1.0029999999999999,"15min":0.997},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.2689999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.81100000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0089999999999999,"15min":1.0149999999999999},"min":{"1min":0.58899999999999997,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.254,"5min":1.292,"15min":1.292},"last":1.1659999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.139,"15min":1.147},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.27}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.91700000000000004,"15min":0.92100000000000004},"min":{"1min":0.68000000000000005,"5min":0.498,"15min":0.436},"max":{"1min":1.4019999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96599999999999997,"15min":0.94299999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":1.0349999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.0469999999999999,"15min":1.0029999999999999},"min":{"1min":0.76100000000000001,"5min":0.61099999999999999,"15min":0.48399999999999999},"max":{"1min":1.446,"5min":1.446,"15min":1.5449999999999999},"last":1.097},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0269999999999999,"15min":1.0600000000000001},"min":{"1min":0.51500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.377,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":1.131}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.0720000000000001,"15min":1.0960000000000001},"min":{"1min":0.76900000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.5069999999999999,"5min":1.577,"15min":1.579},"last":1.069},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.093,"15min":1.0640000000000001},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.53700000000000003},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.242}]}]},{"osd":6,"up_from":42,"seq":180388626660,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302816,"kb_used_data":1624,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442336,"statfs":{"total":95995035648,"available":95684952064,"internally_reserved":0,"allocated":1662976,"data_stored":1170976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.90100000000000002,"15min":0.88500000000000001},"min":{"1min":0.69199999999999995,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.2569999999999999,"5min":1.274,"15min":1.274},"last":1.0649999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.91400000000000003,"15min":0.93600000000000005},"min":{"1min":0.60399999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2030000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.76300000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.88100000000000001,"15min":0.86499999999999999},"min":{"1min":0.63800000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":1.2290000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.89600000000000002,"15min":0.89200000000000002},"min":{"1min":0.626,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":0.89000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:16:16 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.97799999999999998,"15min":0.97399999999999998},"min":{"1min":0.753,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.008,"5min":0.94799999999999995,"15min":0.93799999999999994},"min":{"1min":0.751,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.85399999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:16:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.05,"15min":1.0640000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.204,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.998,"5min":1.01,"15min":1.012},"min":{"1min":0.76300000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.179,"5min":1.272,"15min":1.272},"last":1.1020000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.95099999999999996,"15min":0.93200000000000005},"min":{"1min":0.74099999999999999,"5min":0.63800000000000001,"15min":0.45000000000000001},"max":{"1min":1.3160000000000001,"5min":1.391,"15min":1.5289999999999999},"last":1.002},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0209999999999999,"15min":0.97999999999999998},"min":{"1min":0.73599999999999999,"5min":0.58999999999999997,"15min":0.47399999999999998},"max":{"1min":1.335,"5min":1.335,"15min":1.4590000000000001},"last":1.169}]},{"osd":5,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.038,"15min":1.012},"min":{"1min":0.76200000000000001,"5min":0.63200000000000001,"15min":0.54200000000000004},"max":{"1min":1.2130000000000001,"5min":1.401,"15min":1.708},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0700000000000001,"15min":1.044},"min":{"1min":0.88800000000000001,"5min":0.69199999999999995,"15min":0.50700000000000001},"max":{"1min":1.357,"5min":1.357,"15min":1.4590000000000001},"last":1.294}]},{"osd":7,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.1040000000000001,"15min":1.083},"min":{"1min":0.97699999999999998,"5min":0.71899999999999997,"15min":0.47799999999999998},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.629},"last":1.05},{"interface":"front","average":{"1min":1.196,"5min":1.1419999999999999,"15min":1.1180000000000001},"min":{"1min":0.95599999999999996,"5min":0.72599999999999998,"15min":0.496},"max":{"1min":1.387,"5min":1.5189999999999999,"15min":1.5329999999999999},"last":1.248}]}]},{"osd":1,"up_from":93,"seq":399431958569,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32644,"kb_used_data":1084,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93712508,"statfs":{"total":95995035648,"available":95961608192,"internally_reserved":0,"allocated":1110016,"data_stored":586594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":32301456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.69999999999999996,"15min":0.71599999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.1419999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.68000000000000005,"15min":0.68600000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.98499999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.871}]},{"osd":2,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.67400000000000004,"15min":0.67600000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":1.115},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.82799999999999996,"15min":0.82999999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.71299999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.82499999999999996,"15min":0.83299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.75800000000000001,"15min":0.77400000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0620000000000001,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.748}]},{"osd":4,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.996,"15min":0.97399999999999998},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.98699999999999999,"15min":1.004},"min":{"1min":0.65200000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.464,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.97299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.02,"15min":1.03},"min":{"1min":0.75800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.7390000000000001,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":0.997},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.0620000000000001,"15min":1.042},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.494,"5min":1.589,"15min":1.589},"last":1.0389999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0589999999999999,"15min":1.071},"min":{"1min":0.73599999999999999,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.603,"5min":1.9670000000000001,"15min":1.9670000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.1220000000000001,"15min":1.125},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.877,"5min":1.877,"15min":1.877},"last":0.86199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.133,"15min":1.1259999999999999},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.786,"5min":1.786,"15min":1.786},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.1379999999999999,"15min":1.1679999999999999},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.5580000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":1.091}]}]},{"osd":0,"up_from":88,"seq":377957122095,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37332,"kb_used_data":5772,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707820,"statfs":{"total":95995035648,"available":95956807680,"internally_reserved":0,"allocated":5910528,"data_stored":5371872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":32300288},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.629,"15min":0.60299999999999998},"min":{"1min":0.30399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.73499999999999999,"15min":0.71699999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":1.0189999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92100000000000004,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71299999999999997,"15min":0.71899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.74199999999999999,"15min":0.755},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.96799999999999997,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.76500000000000001,"15min":0.77000000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.66900000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.92600000000000005,"15min":0.93000000000000005},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1619999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.98299999999999998,"15min":0.997},"min":{"1min":0.72399999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.252,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.94999999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0129999999999999,"15min":1.036},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.3280000000000001,"5min":1.575,"15min":1.575},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.0369999999999999,"15min":1.0569999999999999},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.462,"5min":1.472,"15min":1.472},"last":0.80400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0580000000000001,"15min":1.077},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.532,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.0009999999999999,"15min":1.0089999999999999},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.2689999999999999,"5min":1.524,"15min":1.524},"last":0.98399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.1359999999999999,"15min":1.151},"min":{"1min":0.76500000000000001,"5min":0.621,"15min":0.621},"max":{"1min":1.458,"5min":1.667,"15min":1.667},"last":1.085},{"interface":"front","average":{"1min":1.012,"5min":1.097,"15min":1.113},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.325,"5min":1.7,"15min":1.7},"last":1.1120000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795043,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36664,"kb_used_data":5168,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93708488,"statfs":{"total":95995035648,"available":95957491712,"internally_reserved":0,"allocated":5292032,"data_stored":4777297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":32232382},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.65600000000000003,"15min":0.64900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.028,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69899999999999995,"15min":0.69999999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0209999999999999,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.97299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.72999999999999998,"15min":0.72799999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.67600000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.96499999999999997,"15min":0.95499999999999996},"min":{"1min":0.61899999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.264,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.93999999999999995,"15min":0.92700000000000005},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.05}]},{"osd":5,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.006,"15min":1.016},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.3560000000000001,"5min":1.488,"15min":1.488},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.65700000000000003,"5min":0.628,"15min":0.628},"max":{"1min":1.22,"5min":1.27,"15min":1.27},"last":1.0980000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.0029999999999999,"15min":1.0029999999999999},"min":{"1min":0.78800000000000003,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.3720000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.026,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3089999999999999,"5min":1.4670000000000001,"15min":1.4670000000000001},"last":0.872}]},{"osd":7,"last update":"Mon Oct 9 06:16:19 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.091,"15min":1.095},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.389,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0940000000000001,"15min":1.0960000000000001},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.484,"5min":1.54,"15min":1.54},"last":1.1319999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631517,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32424,"kb_used_data":5152,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93712728,"statfs":{"total":95995035648,"available":95961833472,"internally_reserved":0,"allocated":5275648,"data_stored":4775252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":27914785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.66900000000000004,"15min":0.66800000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64600000000000002,"15min":0.64800000000000002},"min":{"1min":0.38800000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84299999999999997,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":1.1499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.70399999999999996,"15min":0.70999999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.98999999999999999,"5min":1.089,"15min":1.089},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.77000000000000002,"15min":0.76200000000000001},"min":{"1min":0.59399999999999997,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.95599999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.82199999999999995,"15min":0.81899999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.127,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.73099999999999998,"15min":0.73399999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.71299999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.91800000000000004,"15min":0.92000000000000004},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2,"5min":1.28,"15min":1.28},"last":1.044},{"interface":"front","average":{"1min":1.02,"5min":1.0029999999999999,"15min":1},"min":{"1min":0.64900000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3100000000000001,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.99099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0329999999999999,"15min":1.0409999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.2589999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.069,"15min":1.0800000000000001},"min":{"1min":0.77000000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.3580000000000001,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.2030000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.1240000000000001,"15min":1.1200000000000001},"min":{"1min":0.80900000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.095,"15min":1.1000000000000001},"min":{"1min":0.745,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.321,"5min":1.577,"15min":1.577},"last":1.109}]},{"osd":7,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0629999999999999,"15min":1.0620000000000001},"min":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"max":{"1min":1.2949999999999999,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1419999999999999,"15min":1.1379999999999999},"min":{"1min":0.93000000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.02}]}]},{"osd":4,"up_from":30,"seq":128849019126,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298152,"kb_used_data":1056,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447000,"statfs":{"total":95995035648,"available":95689728000,"internally_reserved":0,"allocated":1081344,"data_stored":581295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.96799999999999997,"15min":0.96399999999999997},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.198,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.347},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0129999999999999,"15min":1.0249999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3089999999999999,"5min":1.3089999999999999,"15min":1.3089999999999999},"last":1.1140000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96399999999999997,"15min":0.95999999999999996},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.97899999999999998,"15min":0.98399999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.1060000000000001,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.1399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:16:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.91200000000000003,"15min":0.90900000000000003},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.88600000000000001,"15min":0.88400000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.47,"5min":1.47,"15min":1.47},"last":0.96699999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.02,"15min":1.0249999999999999},"min":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.1799999999999999,"5min":1.52,"15min":1.52},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.048,"5min":1.032,"15min":1.0289999999999999},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.0509999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.93300000000000005,"15min":0.90100000000000002},"min":{"1min":0.63900000000000001,"5min":0.59999999999999998,"15min":0.437},"max":{"1min":1.256,"5min":1.4279999999999999,"15min":1.869},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.98499999999999999,"15min":0.97299999999999998},"min":{"1min":0.753,"5min":0.55700000000000005,"15min":0.50600000000000001},"max":{"1min":1.353,"5min":1.4390000000000001,"15min":1.9890000000000001},"last":1.1970000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:16:38 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.0209999999999999,"15min":1.02},"min":{"1min":0.86599999999999999,"5min":0.61599999999999999,"15min":0.48799999999999999},"max":{"1min":1.484,"5min":1.484,"15min":2.0859999999999999},"last":1.014},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0269999999999999,"15min":0.998},"min":{"1min":0.83099999999999996,"5min":0.53000000000000003,"15min":0.436},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.7290000000000001},"last":1.4179999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.0860000000000001,"15min":1.0649999999999999},"min":{"1min":0.92600000000000005,"5min":0.57199999999999995,"15min":0.52200000000000002},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":2.056},"last":1.292},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0529999999999999,"15min":1.0609999999999999},"min":{"1min":0.79700000000000004,"5min":0.60499999999999998,"15min":0.504},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":2.024},"last":1.3740000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822893,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298132,"kb_used_data":1036,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447020,"statfs":{"total":95995035648,"available":95689748480,"internally_reserved":0,"allocated":1060864,"data_stored":576428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.93200000000000005,"15min":0.94499999999999995},"min":{"1min":0.57499999999999996,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.89400000000000002,"15min":0.89200000000000002},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1830000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.67900000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:16:53 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.93000000000000005,"15min":0.93999999999999995},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.155,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.875,"5min":0.90200000000000002,"15min":0.90300000000000002},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.171,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.93500000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.93899999999999995,"15min":0.92300000000000004},"min":{"1min":0.72599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.024,"15min":1.026},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.248}]},{"osd":3,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.018,"15min":1.0089999999999999},"min":{"1min":0.85899999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.294,"5min":1.302,"15min":1.302},"last":1.02},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.0309999999999999,"15min":1.0389999999999999},"min":{"1min":0.74299999999999999,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.1799999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.139}]},{"osd":4,"last update":"Mon Oct 9 06:16:53 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90900000000000003,"15min":0.90300000000000002},"min":{"1min":0.69099999999999995,"5min":0.54400000000000004,"15min":0.42499999999999999},"max":{"1min":1.1519999999999999,"5min":1.399,"15min":1.526},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.008,"15min":0.98999999999999999},"min":{"1min":0.66900000000000004,"5min":0.58199999999999996,"15min":0.49299999999999999},"max":{"1min":1.3620000000000001,"5min":1.4059999999999999,"15min":1.845},"last":1.115}]},{"osd":6,"last update":"Mon Oct 9 06:16:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97699999999999998,"15min":0.98699999999999999},"min":{"1min":0.64700000000000002,"5min":0.56599999999999995,"15min":0.53100000000000003},"max":{"1min":1.091,"5min":1.3460000000000001,"15min":1.4830000000000001},"last":1.216},{"interface":"front","average":{"1min":1.087,"5min":1.1020000000000001,"15min":1.075},"min":{"1min":0.81899999999999995,"5min":0.73399999999999999,"15min":0.51400000000000001},"max":{"1min":1.4059999999999999,"5min":1.4730000000000001,"15min":1.9019999999999999},"last":0.96199999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.133,"15min":1.0960000000000001},"min":{"1min":0.94299999999999995,"5min":0.63700000000000001,"15min":0.52400000000000002},"max":{"1min":1.5860000000000001,"5min":2.2509999999999999,"15min":2.2509999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1399999999999999,"15min":1.125},"min":{"1min":0.95299999999999996,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.387,"5min":2.3969999999999998,"15min":2.3969999999999998},"last":1.3080000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:03.399 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:17:03.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T06:17:04.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[77299]: audit 2023-10-09T06:17:02.456599+0000 mgr.y (mgr.25723) 351 : audit 0 from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:04.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[77299]: cluster 2023-10-09T06:17:02.636579+0000 mgr.y (mgr.25723) 352 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:04.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[77299]: audit 2023-10-09T06:17:03.571281+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:04.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[80002]: audit 2023-10-09T06:17:02.456599+0000 mgr.y (mgr.25723) 351 : audit 0 from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:04.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[80002]: cluster 2023-10-09T06:17:02.636579+0000 mgr.y (mgr.25723) 352 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:03 smithi125 bash[80002]: audit 2023-10-09T06:17:03.571281+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:04.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:03 smithi155 bash[59495]: audit 2023-10-09T06:17:02.456599+0000 mgr.y (mgr.25723) 351 : audit 0 from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:04.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:03 smithi155 bash[59495]: cluster 2023-10-09T06:17:02.636579+0000 mgr.y (mgr.25723) 352 : cluster 0 pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:04.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:03 smithi155 bash[59495]: audit 2023-10-09T06:17:03.571281+0000 mon.a (mon.0) 475 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:06.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:05 smithi125 bash[77299]: cluster 2023-10-09T06:17:04.637389+0000 mgr.y (mgr.25723) 353 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:06.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:05 smithi125 bash[80002]: cluster 2023-10-09T06:17:04.637389+0000 mgr.y (mgr.25723) 353 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:06.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:05 smithi155 bash[59495]: cluster 2023-10-09T06:17:04.637389+0000 mgr.y (mgr.25723) 353 : cluster 0 pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:05.909412+0000 mon.a (mon.0) 476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:05.916572+0000 mon.a (mon.0) 477 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:05.957670+0000 mon.a (mon.0) 478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:05.966044+0000 mon.a (mon.0) 479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:06.446004+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:06.447726+0000 mon.a (mon.0) 481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[77299]: audit 2023-10-09T06:17:06.456868+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:05.909412+0000 mon.a (mon.0) 476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:05.916572+0000 mon.a (mon.0) 477 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:05.957670+0000 mon.a (mon.0) 478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:05.966044+0000 mon.a (mon.0) 479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:06.446004+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:06.447726+0000 mon.a (mon.0) 481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:06 smithi125 bash[80002]: audit 2023-10-09T06:17:06.456868+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:05.909412+0000 mon.a (mon.0) 476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:05.916572+0000 mon.a (mon.0) 477 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:05.957670+0000 mon.a (mon.0) 478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:05.966044+0000 mon.a (mon.0) 479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:06.446004+0000 mon.a (mon.0) 480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:06.447726+0000 mon.a (mon.0) 481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:06 smithi155 bash[59495]: audit 2023-10-09T06:17:06.456868+0000 mon.a (mon.0) 482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:07.683 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:08.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:07 smithi125 bash[80002]: cluster 2023-10-09T06:17:06.638155+0000 mgr.y (mgr.25723) 354 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:08.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:07 smithi125 bash[77299]: cluster 2023-10-09T06:17:06.638155+0000 mgr.y (mgr.25723) 354 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:08.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:07 smithi155 bash[59495]: cluster 2023-10-09T06:17:06.638155+0000 mgr.y (mgr.25723) 354 : cluster 0 pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:09 smithi125 bash[77299]: cluster 2023-10-09T06:17:08.638929+0000 mgr.y (mgr.25723) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:09 smithi125 bash[80002]: cluster 2023-10-09T06:17:08.638929+0000 mgr.y (mgr.25723) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:09.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:09 smithi155 bash[59495]: cluster 2023-10-09T06:17:08.638929+0000 mgr.y (mgr.25723) 355 : cluster 0 pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:11.212 INFO:teuthology.orchestra.run.smithi125.stdout:epoch 104 2023-10-09T06:17:11.212 INFO:teuthology.orchestra.run.smithi125.stdout:fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:created 2023-10-09T05:50:59.273052+0000 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:modified 2023-10-09T06:14:42.656910+0000 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:crush_version 22 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:full_ratio 0.95 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:backfillfull_ratio 0.9 2023-10-09T06:17:11.213 INFO:teuthology.orchestra.run.smithi125.stdout:nearfull_ratio 0.85 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:require_min_compat_client luminous 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:min_compat_client luminous 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:require_osd_release pacific 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:stretch_mode_enabled false 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 82 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T06:17:11.214 INFO:teuthology.orchestra.run.smithi125.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:max_osd 8 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221] [v2:172.21.15.125:6804/155836221,v1:172.21.15.125:6805/155836221] exists,up f01d5c9b-8b14-4090-a4f3-7203497e2ebe 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262] [v2:172.21.15.125:6812/3444691262,v1:172.21.15.125:6813/3444691262] exists,up f3b751bb-a1ce-4054-993d-edd9b0632cc2 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518] [v2:172.21.15.125:6820/1172219518,v1:172.21.15.125:6821/1172219518] exists,up cc437638-50e5-4487-9372-5300b0e63955 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101] [v2:172.21.15.125:6828/2373950101,v1:172.21.15.125:6829/2373950101] exists,up 94e49f30-1edd-4b60-8ac7-1b957d1d2dd9 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050] [v2:172.21.15.155:6802/2095368050,v1:172.21.15.155:6803/2095368050] exists,up 0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641] [v2:172.21.15.155:6810/395277641,v1:172.21.15.155:6811/395277641] exists,up 79c3a7df-d176-4f67-8de0-1bb46949eeba 2023-10-09T06:17:11.215 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062] [v2:172.21.15.155:6818/629807062,v1:172.21.15.155:6819/629807062] exists,up 65cb2125-86cc-4e3c-9bb6-3926ff3db18e 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035] [v2:172.21.15.155:6826/1961936035,v1:172.21.15.155:6827/1961936035] exists,up cd9d3a50-9683-4779-942e-d41abad90f08 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.10 [1,2] 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.17 [1,2] 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:pg_upmap_items 4.1c [1,5] 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2757914842 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1505504676 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/4007395716 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2669712956 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/4000145196 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.216 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:6834/2519956701 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.217 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1272473784 expires 2023-10-10T05:59:24.160468+0000 2023-10-09T06:17:11.217 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/872923687 expires 2023-10-10T05:59:24.160468+0000 2023-10-09T06:17:11.217 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1180505635 expires 2023-10-10T05:59:24.160468+0000 2023-10-09T06:17:11.217 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/928092871 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/457536542 expires 2023-10-10T06:05:34.904573+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/26344715 expires 2023-10-10T06:05:34.904573+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/105318502 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/3175453899 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/26344715 expires 2023-10-10T06:05:34.904573+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/4031419935 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.219 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/872923687 expires 2023-10-10T05:59:24.160468+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2544415312 expires 2023-10-10T05:51:25.812751+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/3780184365 expires 2023-10-10T05:51:55.829150+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/105318502 expires 2023-10-10T06:08:18.401578+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/497767166 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:6835/2519956701 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/3780184365 expires 2023-10-10T05:51:55.829150+0000 2023-10-09T06:17:11.220 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/1760638819 expires 2023-10-10T05:51:25.812751+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/2496192064 expires 2023-10-10T05:51:55.829150+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/4016621680 expires 2023-10-10T06:05:34.904573+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.155:0/3959201196 expires 2023-10-10T06:07:14.674105+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:0/3162814593 expires 2023-10-10T05:51:55.829150+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6800/3168262044 expires 2023-10-10T05:51:25.812751+0000 2023-10-09T06:17:11.221 INFO:teuthology.orchestra.run.smithi125.stdout:blocklist 172.21.15.125:6801/3168262044 expires 2023-10-10T05:51:25.812751+0000 2023-10-09T06:17:11.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:11] "GET /metrics HTTP/1.1" 200 34104 "" "Prometheus/2.33.4" 2023-10-09T06:17:12.006 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:11 smithi125 bash[77299]: cluster 2023-10-09T06:17:10.639801+0000 mgr.y (mgr.25723) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:12.006 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:11 smithi125 bash[77299]: audit 2023-10-09T06:17:11.206228+0000 mon.a (mon.0) 483 : audit 0 from='client.? 172.21.15.125:0/4137558011' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:12.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:11 smithi125 bash[80002]: cluster 2023-10-09T06:17:10.639801+0000 mgr.y (mgr.25723) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:12.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:11 smithi125 bash[80002]: audit 2023-10-09T06:17:11.206228+0000 mon.a (mon.0) 483 : audit 0 from='client.? 172.21.15.125:0/4137558011' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:12.008 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 88 up_thru 103 down_at 85 last_clean_interval [8,84) [v2:172.21.15.125:6802/155836221,v1:172.21.15.125:6803/155836221] [v2:172.21.15.125:6804/155836221,v1:172.21.15.125:6805/155836221] exists,up f01d5c9b-8b14-4090-a4f3-7203497e2ebe', 'osd.1 up in weight 1 up_from 93 up_thru 103 down_at 90 last_clean_interval [13,89) [v2:172.21.15.125:6810/3444691262,v1:172.21.15.125:6811/3444691262] [v2:172.21.15.125:6812/3444691262,v1:172.21.15.125:6813/3444691262] exists,up f3b751bb-a1ce-4054-993d-edd9b0632cc2', 'osd.2 up in weight 1 up_from 98 up_thru 103 down_at 95 last_clean_interval [18,94) [v2:172.21.15.125:6818/1172219518,v1:172.21.15.125:6819/1172219518] [v2:172.21.15.125:6820/1172219518,v1:172.21.15.125:6821/1172219518] exists,up cc437638-50e5-4487-9372-5300b0e63955', 'osd.3 up in weight 1 up_from 103 up_thru 103 down_at 100 last_clean_interval [25,99) [v2:172.21.15.125:6826/2373950101,v1:172.21.15.125:6827/2373950101] [v2:172.21.15.125:6828/2373950101,v1:172.21.15.125:6829/2373950101] exists,up 94e49f30-1edd-4b60-8ac7-1b957d1d2dd9', 'osd.4 up in weight 1 up_from 30 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6800/2095368050,v1:172.21.15.155:6801/2095368050] [v2:172.21.15.155:6802/2095368050,v1:172.21.15.155:6803/2095368050] exists,up 0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd', 'osd.5 up in weight 1 up_from 36 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6808/395277641,v1:172.21.15.155:6809/395277641] [v2:172.21.15.155:6810/395277641,v1:172.21.15.155:6811/395277641] exists,up 79c3a7df-d176-4f67-8de0-1bb46949eeba', 'osd.6 up in weight 1 up_from 42 up_thru 100 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6816/629807062,v1:172.21.15.155:6817/629807062] [v2:172.21.15.155:6818/629807062,v1:172.21.15.155:6819/629807062] exists,up 65cb2125-86cc-4e3c-9bb6-3926ff3db18e', 'osd.7 up in weight 1 up_from 48 up_thru 103 down_at 0 last_clean_interval [0,0) [v2:172.21.15.155:6824/1961936035,v1:172.21.15.155:6825/1961936035] [v2:172.21.15.155:6826/1961936035,v1:172.21.15.155:6827/1961936035] exists,up cd9d3a50-9683-4779-942e-d41abad90f08'] 2023-10-09T06:17:12.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T06:17:12.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:11 smithi155 bash[59495]: cluster 2023-10-09T06:17:10.639801+0000 mgr.y (mgr.25723) 356 : cluster 0 pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:12.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:11 smithi155 bash[59495]: audit 2023-10-09T06:17:11.206228+0000 mon.a (mon.0) 483 : audit 0 from='client.? 172.21.15.125:0/4137558011' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:13 smithi125 bash[77299]: cluster 2023-10-09T06:17:12.640599+0000 mgr.y (mgr.25723) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:14.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:13 smithi125 bash[80002]: cluster 2023-10-09T06:17:12.640599+0000 mgr.y (mgr.25723) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:13 smithi155 bash[59495]: cluster 2023-10-09T06:17:12.640599+0000 mgr.y (mgr.25723) 357 : cluster 0 pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:15 smithi125 bash[77299]: cluster 2023-10-09T06:17:14.641464+0000 mgr.y (mgr.25723) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:15 smithi125 bash[80002]: cluster 2023-10-09T06:17:14.641464+0000 mgr.y (mgr.25723) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:15 smithi155 bash[59495]: cluster 2023-10-09T06:17:14.641464+0000 mgr.y (mgr.25723) 358 : cluster 0 pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:16.782 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:17.976 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:17 smithi125 bash[77299]: cluster 2023-10-09T06:17:16.642240+0000 mgr.y (mgr.25723) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:17.977 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:17 smithi125 bash[80002]: cluster 2023-10-09T06:17:16.642240+0000 mgr.y (mgr.25723) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:18.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:17 smithi155 bash[59495]: cluster 2023-10-09T06:17:16.642240+0000 mgr.y (mgr.25723) 359 : cluster 0 pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:19.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:18 smithi125 bash[77299]: audit 2023-10-09T06:17:18.560773+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:19.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:18 smithi125 bash[80002]: audit 2023-10-09T06:17:18.560773+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:19.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:18 smithi155 bash[59495]: audit 2023-10-09T06:17:18.560773+0000 mon.a (mon.0) 484 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:19.974 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "name": "mon.a", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster": "ceph", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "admin_socket_mode": "", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_client_required": "cephx, none", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_cluster_required": "cephx", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_debug": "true", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T06:17:19.975 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_service_required": "cephx", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "auth_supported": "", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio": "true", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_aio_reap_max": "16", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_async_discard": "false", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_block_size": "4096", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio": "false", 2023-10-09T06:17:19.976 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_enable_discard": "false", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_flock_retry": "3", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_inject_crash": "0", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring": "false", 2023-10-09T06:17:19.977 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring_hipri": "false", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bdev_type": "", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_allocator": "hybrid", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_buffered_io": "true", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T06:17:19.978 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T06:17:19.979 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T06:17:19.980 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_replay_recovery": "false", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluefs_sync_write": "false", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_allocator": "hybrid", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T06:17:19.981 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_create": "true", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_create": "false", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_path": "", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_db_size": "0", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_path": "", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T06:17:19.982 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_size": "107374182400", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_create": "false", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_path": "", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs": "true", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:19.983 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_autotune": "true", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size": "0", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T06:17:19.984 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cache_type": "2q", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_clone_cow": "true", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_mode": "none", 2023-10-09T06:17:19.985 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T06:17:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T06:17:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T06:17:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_freelist": "false", 2023-10-09T06:17:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T06:17:19.986 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T06:17:19.987 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T06:17:19.988 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T06:17:19.989 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fail_eio": "false", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T06:17:19.990 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T06:17:19.991 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T06:17:19.992 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size": "0", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T06:17:19.993 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T06:17:19.994 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_spdk_mem": "512", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T06:17:19.995 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_tracing": "false", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T06:17:19.996 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T06:17:19.997 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_cluster_require_version": "2", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_require_signatures": "false", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_require_version": "2", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_service_require_signatures": "false", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_service_require_version": "2", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "cephx_sign_messages": "true", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "chdir": "", 2023-10-09T06:17:19.998 INFO:teuthology.orchestra.run.smithi125.stdout: "client_acl_type": "", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_asio_thread_count": "2", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_cache_mid": "0.750000", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_cache_size": "16384", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_caps_release_delay": "5", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_check_pool_perm": "true", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_force_sync_read": "false", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_getattr_caps": "false", 2023-10-09T06:17:19.999 INFO:teuthology.orchestra.run.smithi125.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_die_on_failed_remount": "false", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_dirsize_rbytes": "true", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_force_lazyio": "false", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_fs": "", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_inject_release_failure": "false", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_max_inline_size": "4096", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mds_namespace": "", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_metadata": "", 2023-10-09T06:17:20.000 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_gid": "-1", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_timeout": "300", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mount_uid": "-1", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_mountpoint": "/", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_notify_timeout": "10", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc": "true", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_max_objects": "1000", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_size": "209715200", 2023-10-09T06:17:20.001 INFO:teuthology.orchestra.run.smithi125.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_permissions": "true", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_quota": "true", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_quota_df": "true", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_max_bytes": "0", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_max_periods": "4", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_readahead_min": "131072", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_reconnect_stale": "false", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_shutdown_timeout": "30", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_snapdir": ".snap", 2023-10-09T06:17:20.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_tick_interval": "1", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_trace": "", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_use_faked_inos": "false", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "client_use_random_mds": "false", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog": "false", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_graylog_port": "12201", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_monitors": "default=true", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog": "false", 2023-10-09T06:17:20.003 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "clog_to_syslog_level": "info", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_addr": "-", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_network": "", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "cluster_network_interface": "", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_isal": "false", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_level": "5", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "compressor_zstd_level": "1", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T06:17:20.004 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crimson_seastar_smp": "1", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location": "", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location_hook": "", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "crush_location_hook_timeout": "10", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "daemonize": "false", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "daos_pool": "tank", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T06:17:20.005 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_alienstore": "0/5", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asok": "1/5", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asok_assert_abort": "false", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_auth": "1/5", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bdev": "1/3", 2023-10-09T06:17:20.006 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bluefs": "1/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_bluestore": "1/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_buffer": "0/1", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cephsqlite": "0/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_client": "0/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_compressor": "1/5", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_context": "0/1", 2023-10-09T06:17:20.007 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_crush": "1/1", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_crypto": "1/5", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_cyanstore": "0/5", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_dpdk": "1/5", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_eventtrace": "1/5", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_filer": "0/1", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_filestore": "1/3", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_finisher": "1/1", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_fuse": "1/5", 2023-10-09T06:17:20.008 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_javaclient": "1/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_journal": "1/3", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_journaler": "0/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_kstore": "1/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_leveldb": "4/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_lockdep": "0/1", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mclock": "1/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds": "1/5", 2023-10-09T06:17:20.009 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_balancer": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_locker": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_log": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mds_migrator": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_memstore": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mgr": "2/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mgrc": "1/5", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_mon": "20/20", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_monc": "0/10", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_ms": "1/1", 2023-10-09T06:17:20.010 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_none": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objclass": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objectcacher": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_objecter": "0/1", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_optracker": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_osd": "1/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_paxos": "20/20", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_perfcounter": "1/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_prioritycache": "1/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rados": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd": "0/5", 2023-10-09T06:17:20.011 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rbd_replay": "0/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_refs": "0/0", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_reserver": "1/1", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_access": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_flight": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rgw_sync": "1/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_rocksdb": "4/5", 2023-10-09T06:17:20.012 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_backref": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_cache": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_device": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_epm": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_journal": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_lba": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_odata": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_omap": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_onode": "0/5", 2023-10-09T06:17:20.013 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_t": "0/5", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_seastore_tm": "0/5", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_striper": "0/1", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_test": "0/5", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_throttle": "1/1", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_timer": "0/1", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "debug_tp": "0/5", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "device_failure_prediction_mode": "none", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_graylog": "false", 2023-10-09T06:17:20.014 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_journald": "false", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_stderr": "true", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "err_to_syslog": "false", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event_tracing": "false", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_http_port": "9926", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_prio_limit": "5", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_sort_metrics": "true", 2023-10-09T06:17:20.015 INFO:teuthology.orchestra.run.smithi125.stdout: "exporter_stats_period": "5", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "fake_statfs_for_testing": "0", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "fatal_signal_handlers": "true", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filer_max_purge_ops": "10", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filer_max_truncate_ops": "128", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_blackhole": "false", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_btrfs_snap": "true", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_caller_concurrency": "10", 2023-10-09T06:17:20.016 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_omap_check": "false", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_debug_verify_split": "false", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_dump_file": "", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fadvise": "true", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fail_eio": "true", 2023-10-09T06:17:20.017 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fd_cache_size": "128", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fiemap": "false", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_inject_stall": "0", 2023-10-09T06:17:20.018 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_parallel": "false", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_trailing": "false", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_journal_writeahead": "false", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_kill_at": "0", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T06:17:20.019 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T06:17:20.020 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_merge_threshold": "-10", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_odsync_write": "false", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_backend_path": "", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_op_threads": "2", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_punch_hole": "false", 2023-10-09T06:17:20.021 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T06:17:20.022 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_queue_max_ops": "50", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_seek_data_hole": "false", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_sloppy_crc": "false", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_splice": "false", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_split_multiple": "2", 2023-10-09T06:17:20.023 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_split_rand_factor": "20", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_update_to": "1000", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T06:17:20.024 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_xfs_extsize": "false", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "filestore_zfs_snap": "false", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "fio_dir": "/tmp/fio", 2023-10-09T06:17:20.025 INFO:teuthology.orchestra.run.smithi125.stdout: "fsid": "95d0291c-6667-11ee-8db5-212e2dc638e7", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_allow_other": "true", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_big_writes": "true", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_debug": "false", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_default_permissions": "false", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_disable_pagecache": "false", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_max_write": "0", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_multithreaded": "true", 2023-10-09T06:17:20.026 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_require_active_mds": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_set_user_groups": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_move": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_read": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_splice_write": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "gss_target_name": "ceph", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_file": "", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_inject_failure": "0", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "heartbeat_interval": "5", 2023-10-09T06:17:20.027 INFO:teuthology.orchestra.run.smithi125.stdout: "host": "smithi125", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T06:17:20.028 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "inject_early_sigterm": "false", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "jaeger_agent_port": "6799", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "jaeger_tracing_enable": "false", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_aio": "true", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_align_min_size": "65536", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_block_align": "true", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_block_size": "4096", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_dio": "true", 2023-10-09T06:17:20.029 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_discard": "false", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_force_aio": "false", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_ignore_corruption": "false", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_max_write_entries": "100", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_replay_from": "0", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T06:17:20.030 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_write_header_frequency": "0", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "journal_zero_on_create": "false", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_prefetch_periods": "10", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_prezero_periods": "5", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "journaler_write_head_interval": "15", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "key": "", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "keyfile": "", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_backend": "rocksdb", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T06:17:20.031 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_max_bytes": "67108864", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_max_ops": "512", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_onode_map_size": "1024", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "kstore_sync_transaction": "false", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_block_size": "65536", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_bloom_size": "0", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_cache_size": "536870912", 2023-10-09T06:17:20.032 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_compression": "false", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_log": "", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_max_open_files": "0", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_paranoid": "false", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "librados_thread_count": "2", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "lockdep": "false", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "lockdep_force_backtrace": "false", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "log_coarse_timestamps": "true", 2023-10-09T06:17:20.033 INFO:teuthology.orchestra.run.smithi125.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_flush_on_exit": "false", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_graylog_port": "12201", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_max_new": "1000", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_max_recent": "10000", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_stderr_prefix": "debug ", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_file": "true", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_graylog": "false", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_journald": "false", 2023-10-09T06:17:20.034 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_stderr": "false", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "log_to_syslog": "false", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_action_on_write_error": "1", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_alternate_name_max": "8192", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_asio_thread_count": "2", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_export_pin": "true", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T06:17:20.035 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T06:17:20.036 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T06:17:20.036 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T06:17:20.036 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_interval": "10", 2023-10-09T06:17:20.036 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_max": "-1", 2023-10-09T06:17:20.036 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_max_until": "-1", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_merge_size": "50", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T06:17:20.037 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_mode": "0", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_bits": "3", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_size": "10000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T06:17:20.038 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_mid": "0.700000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_interval": "1", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T06:17:20.039 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_auth_pins": "false", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_frag": "false", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_scatterstat": "false", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_debug_subtrees": "false", 2023-10-09T06:17:20.040 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_default_dir_hash": "2", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_defer_session_stale": "true", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_max_entries": "0", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dir_prefetch": "true", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T06:17:20.041 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_early_reply": "true", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_enable_op_tracker": "true", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_enforce_unique_name": "true", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T06:17:20.042 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_health_dummy": "false", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T06:17:20.043 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_join_fs": "", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_journal_format": "1", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_create_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_export_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_import_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T06:17:20.044 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_link_at": "0", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_openc_at": "0", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_rename_at": "0", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_max_events": "-1", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_max_segments": "128", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_pause": "false", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_segment_size": "0", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T06:17:20.045 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_completed_requests": "100000", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_export_size": "20971520", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_file_recover": "32", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_files": "64", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_ops": "8192", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T06:17:20.046 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_metrics_update_interval": "2", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_min_caps_per_client": "100", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_numa_node": "-1", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_history_duration": "600", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_history_size": "20", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_op_log_threshold": "5", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_ping_grace": "15", 2023-10-09T06:17:20.047 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_ping_interval": "5", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_caps": "30000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_replay_interval": "1.000000", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T06:17:20.048 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_root_ino_gid": "0", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_root_ino_uid": "0", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T06:17:20.049 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_shutdown_check": "0", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_skip_ino": "0", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_min_uid": "0", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_snap_rstat": "false", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_symlink_recovery": "true", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_thrash_exports": "0", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_thrash_fragments": "0", 2023-10-09T06:17:20.050 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_tick_interval": "5.000000", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_valgrind_exit": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_verify_backtrace": "1", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_verify_scatter": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mds_wipe_sessions": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mempool_debug": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_page_set": "false", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "memstore_page_size": "65536", 2023-10-09T06:17:20.051 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_bytes": "134217728", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_messages": "512", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_disabled_modules": "", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mds_messages": "128", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T06:17:20.052 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_mon_messages": "128", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_osd_messages": "8192", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_pool": "true", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_standby_modules": "true", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_stats_period": "5", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_stats_threshold": "5", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_tick_period": "2", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T06:17:20.053 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_allow_pool_delete": "true", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_bytes": "104857600", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T06:17:20.054 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T06:17:20.055 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_journald": "false", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_start": "false", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_compact_on_trim": "true", 2023-10-09T06:17:20.056 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_cpu_threads": "4", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_avail_crit": "5", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_avail_warn": "5", 2023-10-09T06:17:20.057 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_json": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_extra_checks": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T06:17:20.058 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_election_default_strategy": "3", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_election_timeout": "5.000000", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_enable_op_tracker": "true", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_fake_pool_delete": "false", 2023-10-09T06:17:20.059 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_force_quorum_join": "false", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_log_update_period": "5", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_max_detail": "50", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog": "true", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_host": "[v2:172.21.15.125:3300/0,v1:172.21.15.125:6789/0] [v2:172.21.15.155:3300/0,v1:172.21.15.155:6789/0] [v2:172.21.15.125:3301/0,v1:172.21.15.125:6790/0]", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_host_override": "", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_initial_members": "", 2023-10-09T06:17:20.060 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease": "5.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_full_interval": "50", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_max": "10000", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_log_max_summary": "50", 2023-10-09T06:17:20.061 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_log_epochs": "500", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_osd": "10000", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T06:17:20.062 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_memory_autotune": "true", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_memory_target": "2147483648", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_digest_period": "5", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_complaint_time": "30", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_duration": "600", 2023-10-09T06:17:20.063 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_size": "20", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_op_log_threshold": "5", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T06:17:20.064 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_cache_size": "500", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T06:17:20.065 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T06:17:20.066 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_report_timeout": "900", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T06:17:20.067 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_max_osds": "4", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T06:17:20.068 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_interval": "86400", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_max_keys": "100", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_scrub_timeout": "300", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_session_timeout": "300", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_smart_report_timeout": "5", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T06:17:20.069 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_pool_size": "4", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_debug": "false", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_tick_interval": "5", 2023-10-09T06:17:20.070 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T06:17:20.071 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_older_version": "true", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T06:17:20.072 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "monmap": "", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T06:17:20.073 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "motr_tracing_enabled": "false", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_op_threads": "3", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_cm": "false", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_device_name": "", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T06:17:20.074 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_sl": "3", 2023-10-09T06:17:20.075 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_rdma_type": "ib", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_async_reap_threshold": "5", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_before_connect": "false", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_ipv4": "true", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_ipv6": "false", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_msgr1": "true", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_msgr2": "true", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_port_max": "7568", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_port_min": "6800", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T06:17:20.076 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_retry_count": "3", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_bind_retry_delay": "5", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_client": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mds": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mgr": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_mon": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_blackhole_osd": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_client_mode": "crc secure", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_cluster_type": "", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_compress_secure": "false", 2023-10-09T06:17:20.077 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_crc_data": "true", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_crc_header": "true", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_bug": "true", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_old_message": "true", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T06:17:20.078 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hugepages": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_lro": "true", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_pmd": "", 2023-10-09T06:17:20.079 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_port_id": "0", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_dump_on_send": "false", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_delay_type": "", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_network_congestion": "0", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_inject_socket_failures": "0", 2023-10-09T06:17:20.080 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_max_accept_failures": "4", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_max_backoff": "15.000000", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compress_mode": "none", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_pq_min_cost": "65536", 2023-10-09T06:17:20.081 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_public_type": "", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_service_mode": "crc secure", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_nodelay": "true", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "ms_type": "async+posix", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "no_config_file": "false", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T06:17:20.082 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inflight_ops": "1024", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "objecter_timeout": "10.000000", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "objectstore_blackhole": "false", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "openssl_engine_opts": "", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_max_ops": "4", 2023-10-09T06:17:20.083 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_agent_slop": "0.020000", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_asio_thread_count": "2", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_scan_max": "512", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backfill_scan_min": "64", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T06:17:20.084 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_peering": "false", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_beacon_report_interval": "300", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_duration": "30", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_blkin_trace_all": "false", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T06:17:20.085 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_class_update_on_start": "true", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_message_cap": "256", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_op_priority": "63", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_client_watch_timeout": "30", 2023-10-09T06:17:20.086 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_command_max_records": "256", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_compact_on_start": "false", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_update_on_start": "true", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T06:17:20.087 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_op_order": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T06:17:20.088 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_shutdown": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T06:17:20.089 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_default_notify_timeout": "30", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T06:17:20.090 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_enable_op_tracker": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_info": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T06:17:20.091 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_function_tracing": "false", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_grace": "20", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_interval": "6", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_stale": "600", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T06:17:20.092 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_journal_size": "5120", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_kill_backfill_at": "0", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_cache_size": "50", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_dedup": "true", 2023-10-09T06:17:20.093 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_message_max": "40", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_attr_name_len": "100", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_attr_size": "0", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_backfills": "1", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_markdown_count": "5", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_markdown_period": "600", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_name_len": "2048", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_object_size": "134217728", 2023-10-09T06:17:20.094 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_pgls": "1024", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_push_cost": "8388608", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_push_objects": "10", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_scrubs": "1", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T06:17:20.095 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_max_write_size": "90", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T06:17:20.096 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_profile": "balanced", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T06:17:20.097 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_base": "805306368", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target": "4294967296", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target_autotune": "false", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_min_recovery_priority": "0", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T06:17:20.098 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_report_interval": "5", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_num_cache_shards": "32", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_node": "-1", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objecter_finishers": "1", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore": "bluestore", 2023-10-09T06:17:20.099 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore_fuse": "false", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_objectstore_tracing": "false", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_duration": "600", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_size": "20", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_log_threshold": "5", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards": "0", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T06:17:20.100 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_op_thread_timeout": "15", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_open_classes_on_start": "true", 2023-10-09T06:17:20.101 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_os_flags": "0", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_peering_op_priority": "255", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_delete_priority": "5", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T06:17:20.102 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_crimson": "false", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T06:17:20.103 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_flags": "0", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_min_size": "0", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T06:17:20.104 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_size": "3", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_default_type": "replicated", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_cost": "20971520", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active": "0", 2023-10-09T06:17:20.105 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_op_priority": "3", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_priority": "5", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T06:17:20.106 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_repair_during_recovery": "false", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T06:17:20.107 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_cost": "52428800", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_end_hour": "0", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T06:17:20.108 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_priority": "5", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T06:17:20.109 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_skip_data_digest": "false", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_smart_report_timeout": "5", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_priority": "5", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T06:17:20.110 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_target_transaction_size": "30", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_tracing": "false", 2023-10-09T06:17:20.111 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_use_stale_snap": "false", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_kill_at": "0", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_max_join_drift": "10", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_min": "500", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_min_wait": "0.050000", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_max": "500", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_service_trim_min": "250", 2023-10-09T06:17:20.112 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_stash_full_interval": "25", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_trim_max": "500", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "paxos_trim_min": "250", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "perf": "true", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "pid_file": "", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "public_addr": "-", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "public_addrv": "", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "public_bind_addr": "-", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "public_network_interface": "", 2023-10-09T06:17:20.113 INFO:teuthology.orchestra.run.smithi125.stdout: "qat_compressor_enabled": "false", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_mon_op_timeout": "0", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_osd_op_timeout": "0", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rados_tracing": "false", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_atime_update_interval": "60", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T06:17:20.114 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache": "true", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_policy": "writearound", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_size": "33554432", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T06:17:20.115 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_compression_hint": "none", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_clone_format": "auto", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_data_pool": "", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_features": "61", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_format": "2", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_map_options": "", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_order": "22", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_pool": "rbd", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_stripe_count": "0", 2023-10-09T06:17:20.116 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_io_scheduler": "simple", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T06:17:20.117 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_order": "24", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_pool": "", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_journal_splay_width": "4", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T06:17:20.118 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T06:17:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T06:17:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T06:17:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T06:17:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T06:17:20.119 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T06:17:20.120 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T06:17:20.121 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_op_threads": "1", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T06:17:20.122 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_plugins": "", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T06:17:20.123 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T06:17:20.124 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_tracing": "false", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_validate_names": "true", 2023-10-09T06:17:20.125 INFO:teuthology.orchestra.run.smithi125.stdout: "rbd_validate_pool": "true", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "restapi_base_url": "", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "restapi_log_level": "", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_admin_entry": "admin", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_backend_store": "rados", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_barbican_url": "", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_beast_enable_async": "true", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T06:17:20.126 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_enabled": "true", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_config_store": "rados", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_content_length_compat": "false", 2023-10-09T06:17:20.127 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_cross_domain_policy": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T06:17:20.128 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T06:17:20.129 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T06:17:20.130 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T06:17:20.131 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T06:17:20.132 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_log_window": "30", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T06:17:20.133 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T06:17:20.134 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dns_name": "", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dns_s3website_name": "", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_ops_log": "false", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T06:17:20.135 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_static_website": "false", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enable_usage_log": "false", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_expose_bucket": "false", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_extended_http_attrs": "", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_filter": "none", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T06:17:20.136 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_objs": "32", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T06:17:20.137 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_init_timeout": "300", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_password": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_project": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_token": "", 2023-10-09T06:17:20.138 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_admin_user": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_api_version": "2", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T06:17:20.139 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_url": "", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_objs": "32", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_worker": "3", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T06:17:20.140 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_http_headers": "", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T06:17:20.141 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attr_size": "0", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T06:17:20.142 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_listing_results": "1000", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_notify_retries": "10", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T06:17:20.143 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T06:17:20.144 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_num_control_oids": "8", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_numa_node": "-1", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T06:17:20.145 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_op_tracing": "false", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_token": "", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_url": "", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T06:17:20.146 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_rados": "false", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_print_continue": "true", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T06:17:20.147 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rados_tracing": "false", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm": "", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm_id": "", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_region": "", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T06:17:20.148 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_request_uri": "", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_resolve_cname": "false", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_run_sync_thread": "true", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T06:17:20.149 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_script_uri": "", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_service_provider_name": "", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T06:17:20.150 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_client_id": "", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_client_secret": "", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_entry": "sts", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_key": "sts", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_auth_url": "", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_custom_header": "", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T06:17:20.151 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_need_stats": "true", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_tenant_name": "", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_url": "", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_lease_period": "120", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T06:17:20.152 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_thread_pool_size": "512", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_comment": "", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_createby": "", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_encoding": "", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_flag": "false", 2023-10-09T06:17:20.153 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_origin": "", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_torrent_tracker": "", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_max_shards": "32", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_use_opa_authz": "false", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T06:17:20.154 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_user_unique_email": "true", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_verify_ssl": "true", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone": "", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone_id": "", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T06:17:20.155 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup": "", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup_id": "", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_block_size": "4096", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T06:17:20.156 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T06:17:20.157 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_partition_filters": "false", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_perf": "false", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "run_dir": "/var/run/ceph", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_block_create": "true", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T06:17:20.158 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_device_size": "53687091200", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_main_device_type": "SSD", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "seastore_segment_size": "67108864", 2023-10-09T06:17:20.159 INFO:teuthology.orchestra.run.smithi125.stdout: "set_keepcaps": "false", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "setgroup": "ceph", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "setuser": "ceph", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "setuser_match_path": "", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "thp": "false", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "threadpool_default_timeout": "60", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout: "throttler_perf_counter": "true" 2023-10-09T06:17:20.160 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:17:20.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:19 smithi155 bash[59495]: cluster 2023-10-09T06:17:18.642975+0000 mgr.y (mgr.25723) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:20.161 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:19 smithi125 bash[77299]: cluster 2023-10-09T06:17:18.642975+0000 mgr.y (mgr.25723) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:20.161 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:19 smithi125 bash[80002]: cluster 2023-10-09T06:17:18.642975+0000 mgr.y (mgr.25723) 360 : cluster 0 pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:20.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T06:17:21.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:21] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.33.4" 2023-10-09T06:17:22.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:21 smithi125 bash[77299]: cluster 2023-10-09T06:17:20.643829+0000 mgr.y (mgr.25723) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:22.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:21 smithi125 bash[80002]: cluster 2023-10-09T06:17:20.643829+0000 mgr.y (mgr.25723) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:22.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:21 smithi155 bash[59495]: cluster 2023-10-09T06:17:20.643829+0000 mgr.y (mgr.25723) 361 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:24.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:23 smithi125 bash[77299]: cluster 2023-10-09T06:17:22.644614+0000 mgr.y (mgr.25723) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:24.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:23 smithi125 bash[80002]: cluster 2023-10-09T06:17:22.644614+0000 mgr.y (mgr.25723) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:23 smithi155 bash[59495]: cluster 2023-10-09T06:17:22.644614+0000 mgr.y (mgr.25723) 362 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:25.613 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:26.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:25 smithi125 bash[80002]: cluster 2023-10-09T06:17:24.645510+0000 mgr.y (mgr.25723) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:26.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:25 smithi125 bash[77299]: cluster 2023-10-09T06:17:24.645510+0000 mgr.y (mgr.25723) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:26.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:25 smithi155 bash[59495]: cluster 2023-10-09T06:17:24.645510+0000 mgr.y (mgr.25723) 363 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:28.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:27 smithi125 bash[77299]: cluster 2023-10-09T06:17:26.646283+0000 mgr.y (mgr.25723) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:28.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:27 smithi125 bash[80002]: cluster 2023-10-09T06:17:26.646283+0000 mgr.y (mgr.25723) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:28.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:27 smithi155 bash[59495]: cluster 2023-10-09T06:17:26.646283+0000 mgr.y (mgr.25723) 364 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:28.804 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a: {} 2023-10-09T06:17:28.821 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b: {} 2023-10-09T06:17:28.839 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c: {} 2023-10-09T06:17:28.840 INFO:teuthology.orchestra.run.smithi125.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:28.840 INFO:teuthology.orchestra.run.smithi125.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:28.840 INFO:teuthology.orchestra.run.smithi125.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:29.635 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T06:17:29.636 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T06:17:29.641 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:17:29.642 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:17:29.642 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T06:17:29.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T06:17:29.643 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T06:17:29.643 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:17:29.643 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:17:29.644 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T06:17:29.644 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:17:29.644 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:17:29.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T06:17:29.644 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T06:17:29.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:29.645 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T06:17:29.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:17:29.645 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T06:17:29.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:17:29.646 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T06:17:29.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:17:29.681 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:29.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:29.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:29.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:29.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:29.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:29 smithi125 bash[77299]: cluster 2023-10-09T06:17:28.646944+0000 mgr.y (mgr.25723) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:29 smithi125 bash[77299]: audit 2023-10-09T06:17:28.782770+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.125:0/2040080665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:29 smithi125 bash[80002]: cluster 2023-10-09T06:17:28.646944+0000 mgr.y (mgr.25723) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:29 smithi125 bash[80002]: audit 2023-10-09T06:17:28.782770+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.125:0/2040080665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:29.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:29 smithi155 bash[59495]: cluster 2023-10-09T06:17:28.646944+0000 mgr.y (mgr.25723) 365 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:29 smithi155 bash[59495]: audit 2023-10-09T06:17:28.782770+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.125:0/2040080665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:29.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:29.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:29.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:30.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:30.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:30.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:30.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:30.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:30.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:30.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:30.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:30.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:30.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:30.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:30.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:30.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:30.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:30.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:30.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:31.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:31.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:31.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:31.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:31.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:31.446 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:31.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:31.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:31.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:31] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.33.4" 2023-10-09T06:17:31.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:31.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:31.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:31.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:31.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:31 smithi125 bash[77299]: cluster 2023-10-09T06:17:30.647775+0000 mgr.y (mgr.25723) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:31.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:31 smithi125 bash[80002]: cluster 2023-10-09T06:17:30.647775+0000 mgr.y (mgr.25723) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:31.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:31.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:32.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:32.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:32.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:31 smithi155 bash[59495]: cluster 2023-10-09T06:17:30.647775+0000 mgr.y (mgr.25723) 366 : cluster 0 pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:32.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:32.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:32.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:32.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:32.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:32.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:32.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:32.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:32.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:32.782 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:32.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:32.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:32.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:32.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:33.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:33.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:33.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:33.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:33.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:33.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:33.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.745 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:33 smithi155 bash[59495]: cluster 2023-10-09T06:17:32.648649+0000 mgr.y (mgr.25723) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:33.745 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:33 smithi155 bash[59495]: audit 2023-10-09T06:17:33.560835+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:33.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:33.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:33.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:33.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:34.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:34.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:33 smithi125 bash[77299]: cluster 2023-10-09T06:17:32.648649+0000 mgr.y (mgr.25723) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:34.017 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:33 smithi125 bash[77299]: audit 2023-10-09T06:17:33.560835+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:34.017 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:33 smithi125 bash[80002]: cluster 2023-10-09T06:17:32.648649+0000 mgr.y (mgr.25723) 367 : cluster 0 pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:34.018 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:33 smithi125 bash[80002]: audit 2023-10-09T06:17:33.560835+0000 mon.a (mon.0) 485 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:34.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:34.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:34.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:34.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:34.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:34.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.425 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:17:34.427 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:34.427 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:34.428 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:34.430 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:34.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:34.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:34.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:34.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:34.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:34.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:34.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:34.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:35.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:35.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:35.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:35.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:35.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:35.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:35.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:35.465 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:35.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:35.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:35.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:35.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:35.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:35.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:35.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:35 smithi155 bash[59495]: cluster 2023-10-09T06:17:34.649612+0000 mgr.y (mgr.25723) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:35.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:35 smithi125 bash[80002]: cluster 2023-10-09T06:17:34.649612+0000 mgr.y (mgr.25723) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:35.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:35 smithi125 bash[77299]: cluster 2023-10-09T06:17:34.649612+0000 mgr.y (mgr.25723) 368 : cluster 0 pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:35.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:35.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:35.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:36.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:36.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:36.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:36.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:36.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:36.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:36.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:36.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:36.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:36.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:36.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:36.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:36.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:37.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:37.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:37.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:37.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:37.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:37.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:37.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:37.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:37.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:37.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:37.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:37.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:37.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:37.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:37.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:37 smithi125 bash[77299]: cluster 2023-10-09T06:17:36.650445+0000 mgr.y (mgr.25723) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:37.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:37 smithi125 bash[80002]: cluster 2023-10-09T06:17:36.650445+0000 mgr.y (mgr.25723) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:37.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:37.973 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:17:37.973 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:17:37.974 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:17:37.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:37 smithi155 bash[59495]: cluster 2023-10-09T06:17:36.650445+0000 mgr.y (mgr.25723) 369 : cluster 0 pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:37.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:37.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:38.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:38.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:38.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:38.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:38.250 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:38.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:38.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:38.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:38.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:38.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:38.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:38.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:38.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:38.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:38.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:38.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:38.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:39.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:39.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:39.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:39.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:39.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:39.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:39.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:39.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:39.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:39.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:39.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:39.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:39.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:39.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:39.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:39.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:39 smithi125 bash[77299]: cluster 2023-10-09T06:17:38.650934+0000 mgr.y (mgr.25723) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:39.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:39 smithi125 bash[80002]: cluster 2023-10-09T06:17:38.650934+0000 mgr.y (mgr.25723) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:39.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:39.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:39 smithi155 bash[59495]: cluster 2023-10-09T06:17:38.650934+0000 mgr.y (mgr.25723) 370 : cluster 0 pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:39.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:39.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:39.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:40.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:40.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:40.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:40.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:40.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:40.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:40.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:40.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:40.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:40.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:40.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:40.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:40.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:40.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:40.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:40.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:41.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:41.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:41.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:41.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:41.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:41.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:41.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:41.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:41.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:41.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:41.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:41] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.33.4" 2023-10-09T06:17:41.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:41.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:41.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:41.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:41.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:42.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:42.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:41 smithi155 bash[59495]: cluster 2023-10-09T06:17:40.651725+0000 mgr.y (mgr.25723) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:42.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:42.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:41 smithi125 bash[77299]: cluster 2023-10-09T06:17:40.651725+0000 mgr.y (mgr.25723) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:42.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:41 smithi125 bash[80002]: cluster 2023-10-09T06:17:40.651725+0000 mgr.y (mgr.25723) 371 : cluster 0 pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:42.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:42.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:42.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:42.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:42.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:42.481 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:42.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:42.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:42.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:42.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:42.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:42.768 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:17:42.788 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:17:42.810 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:17:42.835 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:17:42.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:42.858 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:17:42.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:42.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:42.887 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:17:42.910 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:17:42.937 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:17:42.939 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.939 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.939 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.939 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.939 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.940 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.940 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.940 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:17:42.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:42.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:43.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:43.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:43.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:43.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:43.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:43.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:43.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:43.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:43.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:43.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:43.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:43.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:43.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:43.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:43.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:43.779 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:17:43.780 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:17:43.780 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:17:43.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T06:17:43.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:43.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:43.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:44.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:44.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:44.110 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:17:44.118 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_0' created 2023-10-09T06:17:44.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:44 smithi155 bash[59495]: cluster 2023-10-09T06:17:42.652215+0000 mgr.y (mgr.25723) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:44 smithi155 bash[59495]: audit 2023-10-09T06:17:42.741629+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.125:0/805146824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:44 smithi155 bash[59495]: audit 2023-10-09T06:17:43.202487+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:44.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:44.260 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:17:44.261 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:17:44.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:44.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:44.361 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[77299]: cluster 2023-10-09T06:17:42.652215+0000 mgr.y (mgr.25723) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:44.361 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[77299]: audit 2023-10-09T06:17:42.741629+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.125:0/805146824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.361 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[77299]: audit 2023-10-09T06:17:43.202487+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:44.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[80002]: cluster 2023-10-09T06:17:42.652215+0000 mgr.y (mgr.25723) 372 : cluster 0 pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:44.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[80002]: audit 2023-10-09T06:17:42.741629+0000 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.125:0/805146824' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:44.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:44 smithi125 bash[80002]: audit 2023-10-09T06:17:43.202487+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:44.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:44.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:44.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:44.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:44.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:44.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:44.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:44.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:44.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:44.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:44.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:44.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:44.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:44.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:45.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:45.113 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:45.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:45.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:45.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:45.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: audit 2023-10-09T06:17:43.632361+0000 mon.a (mon.0) 488 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: cluster 2023-10-09T06:17:44.083767+0000 mon.a (mon.0) 489 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: audit 2023-10-09T06:17:44.097981+0000 mon.a (mon.0) 490 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: audit 2023-10-09T06:17:44.098113+0000 mon.a (mon.0) 491 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: cluster 2023-10-09T06:17:44.098207+0000 mon.a (mon.0) 492 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: cluster 2023-10-09T06:17:44.100027+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: cluster 2023-10-09T06:17:44.102366+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.363 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[80002]: cluster 2023-10-09T06:17:44.653169+0000 mgr.y (mgr.25723) 373 : cluster 0 pgmap v307: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: audit 2023-10-09T06:17:43.632361+0000 mon.a (mon.0) 488 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: cluster 2023-10-09T06:17:44.083767+0000 mon.a (mon.0) 489 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: audit 2023-10-09T06:17:44.097981+0000 mon.a (mon.0) 490 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: audit 2023-10-09T06:17:44.098113+0000 mon.a (mon.0) 491 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: cluster 2023-10-09T06:17:44.098207+0000 mon.a (mon.0) 492 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: cluster 2023-10-09T06:17:44.100027+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: cluster 2023-10-09T06:17:44.102366+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:45 smithi125 bash[77299]: cluster 2023-10-09T06:17:44.653169+0000 mgr.y (mgr.25723) 373 : cluster 0 pgmap v307: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:45.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:45.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:45.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: audit 2023-10-09T06:17:43.632361+0000 mon.a (mon.0) 488 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: cluster 2023-10-09T06:17:44.083767+0000 mon.a (mon.0) 489 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: audit 2023-10-09T06:17:44.097981+0000 mon.a (mon.0) 490 : audit 1 from='client.? 172.21.15.125:0/457572505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: audit 2023-10-09T06:17:44.098113+0000 mon.a (mon.0) 491 : audit 1 from='client.? 172.21.15.125:0/67522252' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: cluster 2023-10-09T06:17:44.098207+0000 mon.a (mon.0) 492 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: cluster 2023-10-09T06:17:44.100027+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: cluster 2023-10-09T06:17:44.102366+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e105 with expected crc 2023-10-09T06:17:45.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:45 smithi155 bash[59495]: cluster 2023-10-09T06:17:44.653169+0000 mgr.y (mgr.25723) 373 : cluster 0 pgmap v307: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:45.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:45.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:45.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:45.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:45.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:45.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:45.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:45.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:46.023 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:46.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:46.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T06:17:46.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:46.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:46.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:46.310 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[77299]: cluster 2023-10-09T06:17:45.104165+0000 mon.a (mon.0) 493 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[77299]: cluster 2023-10-09T06:17:45.108382+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[77299]: cluster 2023-10-09T06:17:45.108479+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[77299]: cluster 2023-10-09T06:17:45.108592+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[77299]: cluster 2023-10-09T06:17:45.108903+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[80002]: cluster 2023-10-09T06:17:45.104165+0000 mon.a (mon.0) 493 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:17:46.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[80002]: cluster 2023-10-09T06:17:45.108382+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[80002]: cluster 2023-10-09T06:17:45.108479+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[80002]: cluster 2023-10-09T06:17:45.108592+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.312 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:46 smithi125 bash[80002]: cluster 2023-10-09T06:17:45.108903+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:46.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:46 smithi155 bash[59495]: cluster 2023-10-09T06:17:45.104165+0000 mon.a (mon.0) 493 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:17:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:46 smithi155 bash[59495]: cluster 2023-10-09T06:17:45.108382+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:46 smithi155 bash[59495]: cluster 2023-10-09T06:17:45.108479+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:46 smithi155 bash[59495]: cluster 2023-10-09T06:17:45.108592+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:46 smithi155 bash[59495]: cluster 2023-10-09T06:17:45.108903+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e106 with expected crc 2023-10-09T06:17:46.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:46.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:46.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:17:46.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:46.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:46.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:17:46.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:46.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:46.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:46.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:46.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:46.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:46.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:46.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:47.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:47.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:47.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:47.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:47.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:47.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:47.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:47 smithi155 bash[59495]: cluster 2023-10-09T06:17:46.107772+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T06:17:47.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:47 smithi155 bash[59495]: cluster 2023-10-09T06:17:46.108225+0000 mon.a (mon.0) 494 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T06:17:47.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:47 smithi155 bash[59495]: cluster 2023-10-09T06:17:46.653827+0000 mgr.y (mgr.25723) 374 : cluster 0 pgmap v310: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:47.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[77299]: cluster 2023-10-09T06:17:46.107772+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T06:17:47.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[77299]: cluster 2023-10-09T06:17:46.108225+0000 mon.a (mon.0) 494 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T06:17:47.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[77299]: cluster 2023-10-09T06:17:46.653827+0000 mgr.y (mgr.25723) 374 : cluster 0 pgmap v310: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:47.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[80002]: cluster 2023-10-09T06:17:46.107772+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-09T06:17:47.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[80002]: cluster 2023-10-09T06:17:46.108225+0000 mon.a (mon.0) 494 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-09T06:17:47.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:47 smithi125 bash[80002]: cluster 2023-10-09T06:17:46.653827+0000 mgr.y (mgr.25723) 374 : cluster 0 pgmap v310: 89 pgs: 7 creating+peering, 2 unknown, 80 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:47.407 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:47.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:47.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:47.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:47.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:47.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:47.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:47.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:47.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:47.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:48.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:48.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:48.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:48.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:48.331 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:48.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:48.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:48.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:48.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:48.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:17:48.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:48.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:48.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:48.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:48.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:49.135 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:49.168 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:49.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:49.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:49.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:49.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:49.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:49.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:49.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:49.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:49.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:49.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[77299]: audit 2023-10-09T06:17:48.567765+0000 mon.a (mon.0) 495 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:49.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[77299]: audit 2023-10-09T06:17:48.569408+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:49.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[77299]: cluster 2023-10-09T06:17:48.654393+0000 mgr.y (mgr.25723) 375 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:49.833 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[80002]: audit 2023-10-09T06:17:48.567765+0000 mon.a (mon.0) 495 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:49.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[80002]: audit 2023-10-09T06:17:48.569408+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:49.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:49 smithi125 bash[80002]: cluster 2023-10-09T06:17:48.654393+0000 mgr.y (mgr.25723) 375 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:49.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:49.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:49 smithi155 bash[59495]: audit 2023-10-09T06:17:48.567765+0000 mon.a (mon.0) 495 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:49 smithi155 bash[59495]: audit 2023-10-09T06:17:48.569408+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:49 smithi155 bash[59495]: cluster 2023-10-09T06:17:48.654393+0000 mgr.y (mgr.25723) 375 : cluster 0 pgmap v311: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:49.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:49.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:50.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:50.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:50.173 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:50.174 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:50.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:50.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:50.364 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:50.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:50.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:50.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:50.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:50.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:50.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:50.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:50.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:50.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:50.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:50.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:50.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:51.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:51.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:51.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:51.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:51.187 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:51.188 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:51.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:51.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:51.301 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:17:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:17:51] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T06:17:51.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:51.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:51.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:51.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:51.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:51.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:51.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:51.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:51.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:51.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:51.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:51.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:51.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:51 smithi155 bash[59495]: cluster 2023-10-09T06:17:50.655196+0000 mgr.y (mgr.25723) 376 : cluster 0 pgmap v312: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:51.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:52.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.094 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:51 smithi125 bash[77299]: cluster 2023-10-09T06:17:50.655196+0000 mgr.y (mgr.25723) 376 : cluster 0 pgmap v312: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:52.094 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:51 smithi125 bash[80002]: cluster 2023-10-09T06:17:50.655196+0000 mgr.y (mgr.25723) 376 : cluster 0 pgmap v312: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:52.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:52.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:52.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:52.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:52.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:52.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:52.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:52.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:52.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:52.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:52.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:52.889 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:52.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:52.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:52.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:53.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:53.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:53.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:53.222 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:53.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:53.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:53.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:53.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:53.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:53.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:53.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:53.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:53.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:53.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:53.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:53 smithi155 bash[59495]: cluster 2023-10-09T06:17:52.656018+0000 mgr.y (mgr.25723) 377 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:53.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:53.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:53.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:54.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:54.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:53 smithi125 bash[80002]: cluster 2023-10-09T06:17:52.656018+0000 mgr.y (mgr.25723) 377 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:54.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:53 smithi125 bash[77299]: cluster 2023-10-09T06:17:52.656018+0000 mgr.y (mgr.25723) 377 : cluster 0 pgmap v313: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:54.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:54.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:54.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:54.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:54.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:54.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:54.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:54.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:54.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:54.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:54.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:54.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:54.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:55.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:55.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:55.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:55.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:55.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:55.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:55.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:55.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:55.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:55.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:55.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:55.670 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:55.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:55.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:55.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:55 smithi125 bash[80002]: cluster 2023-10-09T06:17:54.656743+0000 mgr.y (mgr.25723) 378 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:55.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:55 smithi125 bash[77299]: cluster 2023-10-09T06:17:54.656743+0000 mgr.y (mgr.25723) 378 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:55.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:55.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:55.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:55.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:56.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:56.057 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:55 smithi155 bash[59495]: cluster 2023-10-09T06:17:54.656743+0000 mgr.y (mgr.25723) 378 : cluster 0 pgmap v314: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:56.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:56.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:56.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:56.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:56.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:56.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:56.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:56.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:56.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:56.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:56.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:56.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:56.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:56.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:57.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:57.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:57.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:57.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:57.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:57.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:57.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:57.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:57.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:57.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:57.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:58.576 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:57 smithi155 bash[59495]: cluster 2023-10-09T06:17:56.657643+0000 mgr.y (mgr.25723) 379 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:58.626 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:57 smithi125 bash[77299]: cluster 2023-10-09T06:17:56.657643+0000 mgr.y (mgr.25723) 379 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:58.626 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:57 smithi125 bash[80002]: cluster 2023-10-09T06:17:56.657643+0000 mgr.y (mgr.25723) 379 : cluster 0 pgmap v315: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:58.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:58.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:58.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:58.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:58.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:59.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:59.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:59.168 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:59.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:59.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:59.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:59.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:59.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:59.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:59.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:59.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:59.800 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:59 smithi155 bash[59495]: cluster 2023-10-09T06:17:58.658224+0000 mgr.y (mgr.25723) 380 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:59.800 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:17:59 smithi155 bash[59495]: audit 2023-10-09T06:17:58.759062+0000 mon.a (mon.0) 497 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:59.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:59.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:59.937 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:59 smithi125 bash[80002]: cluster 2023-10-09T06:17:58.658224+0000 mgr.y (mgr.25723) 380 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:59.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:17:59 smithi125 bash[80002]: audit 2023-10-09T06:17:58.759062+0000 mon.a (mon.0) 497 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:59.938 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:59 smithi125 bash[77299]: cluster 2023-10-09T06:17:58.658224+0000 mgr.y (mgr.25723) 380 : cluster 0 pgmap v316: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:17:59.938 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:17:59 smithi125 bash[77299]: audit 2023-10-09T06:17:58.759062+0000 mon.a (mon.0) 497 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:17:59.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:17:59.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:00.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:00.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:00.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:00.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:00.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:00.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:00.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:00.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:00.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:00.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:00.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:00.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:00.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:00.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:00.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:00.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:00.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:00.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:01.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:01.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:01.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:01.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:01.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:01.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:01.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:01] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-10-09T06:18:01.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:01.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:01.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:01.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:01.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:01.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:01.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:01.784 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.7. 2023-10-09T06:18:01.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:01 smithi155 bash[59495]: cluster 2023-10-09T06:18:00.659224+0000 mgr.y (mgr.25723) 381 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:18:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:01 smithi155 bash[59495]: audit 2023-10-09T06:18:01.657462+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.125:0/426157862' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:01 smithi155 bash[59495]: audit 2023-10-09T06:18:01.657964+0000 mon.a (mon.0) 498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:01 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.658090+0000 mon.a (mon.0) 499 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:01.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:01.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:01.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:02.010 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:18:02.034 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:18:02.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[80002]: cluster 2023-10-09T06:18:00.659224+0000 mgr.y (mgr.25723) 381 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[80002]: audit 2023-10-09T06:18:01.657462+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.125:0/426157862' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[80002]: audit 2023-10-09T06:18:01.657964+0000 mon.a (mon.0) 498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.658090+0000 mon.a (mon.0) 499 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[77299]: debug 2023-10-09T06:18:01.653+0000 7ff949fd9700 -1 mon.a@0(leader).osd e107 definitely_dead 0 2023-10-09T06:18:02.050 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[77299]: cluster 2023-10-09T06:18:00.659224+0000 mgr.y (mgr.25723) 381 : cluster 0 pgmap v317: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-09T06:18:02.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[77299]: audit 2023-10-09T06:18:01.657462+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.125:0/426157862' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:02.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[77299]: audit 2023-10-09T06:18:01.657964+0000 mon.a (mon.0) 498 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:02.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:01 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.658090+0000 mon.a (mon.0) 499 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:02.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:02.066 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:18:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:02.084 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:18:02.112 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:18:02.142 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:18:02.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:02.168 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:18:02.194 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:18:02.196 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.196 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.196 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.197 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:02.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:02.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:02.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:02.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:02.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:02.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:02.606 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:02.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:02.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:02.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:02.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:02.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:02.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:03.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:03.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: audit 2023-10-09T06:18:01.770093+0000 mon.a (mon.0) 500 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:03.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.770162+0000 mon.a (mon.0) 501 : cluster 0 osdmap e108: 8 total, 8 up, 7 in 2023-10-09T06:18:03.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.772432+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.774519+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.774953+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.775862+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.777109+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.780124+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.781717+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: cluster 2023-10-09T06:18:01.783401+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[80002]: audit 2023-10-09T06:18:01.979973+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.125:0/866697152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:03.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: audit 2023-10-09T06:18:01.770093+0000 mon.a (mon.0) 500 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.770162+0000 mon.a (mon.0) 501 : cluster 0 osdmap e108: 8 total, 8 up, 7 in 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.772432+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.774519+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.774953+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.775862+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.777109+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.780124+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.781717+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: cluster 2023-10-09T06:18:01.783401+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:02 smithi125 bash[77299]: audit 2023-10-09T06:18:01.979973+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.125:0/866697152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: audit 2023-10-09T06:18:01.770093+0000 mon.a (mon.0) 500 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.770162+0000 mon.a (mon.0) 501 : cluster 0 osdmap e108: 8 total, 8 up, 7 in 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.772432+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.774519+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.774953+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.775862+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.777109+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.780124+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.781717+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: cluster 2023-10-09T06:18:01.783401+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e108 with expected crc 2023-10-09T06:18:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:02 smithi155 bash[59495]: audit 2023-10-09T06:18:01.979973+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.125:0/866697152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:03.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:03.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:03.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:03.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:03.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:03.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:03.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:03.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:03.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:03.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.804 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.659906+0000 mgr.y (mgr.25723) 382 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail 2023-10-09T06:18:03.804 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.780598+0000 mon.a (mon.0) 502 : cluster 0 osdmap e109: 8 total, 8 up, 7 in 2023-10-09T06:18:03.804 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.781982+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.804 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.785115+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.805 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.785161+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.805 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.786742+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.805 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.787841+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.805 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.788421+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.805 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.789807+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.806 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: cluster 2023-10-09T06:18:02.790054+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.806 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[77299]: audit 2023-10-09T06:18:03.561685+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:03.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:03.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:03.943 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.659906+0000 mgr.y (mgr.25723) 382 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.780598+0000 mon.a (mon.0) 502 : cluster 0 osdmap e109: 8 total, 8 up, 7 in 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.781982+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.785115+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.785161+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.786742+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.944 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.787841+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.788421+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.789807+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: cluster 2023-10-09T06:18:02.790054+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:03 smithi155 bash[59495]: audit 2023-10-09T06:18:03.561685+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:03.951 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.659906+0000 mgr.y (mgr.25723) 382 : cluster 0 pgmap v319: 89 pgs: 89 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail 2023-10-09T06:18:03.951 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.780598+0000 mon.a (mon.0) 502 : cluster 0 osdmap e109: 8 total, 8 up, 7 in 2023-10-09T06:18:03.951 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.781982+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.951 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.785115+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.951 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.785161+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.786742+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.787841+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.788421+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.789807+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: cluster 2023-10-09T06:18:02.790054+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e109 with expected crc 2023-10-09T06:18:03.952 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:03 smithi125 bash[80002]: audit 2023-10-09T06:18:03.561685+0000 mon.a (mon.0) 503 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:03.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:03.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:04.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:04.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:04.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:04.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:04.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:04.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:04.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:04.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:04.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:04.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:04.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:04.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:04.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:04.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:04.883 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.797041+0000 mon.a (mon.0) 504 : cluster 0 osdmap e110: 8 total, 8 up, 7 in 2023-10-09T06:18:04.883 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.800961+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.883 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.801395+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.883 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.803083+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.804989+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.805160+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.805736+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.807104+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.810076+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[80002]: cluster 2023-10-09T06:18:03.810190+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.797041+0000 mon.a (mon.0) 504 : cluster 0 osdmap e110: 8 total, 8 up, 7 in 2023-10-09T06:18:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.800961+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.801395+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.803083+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.804989+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.805160+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.805736+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.807104+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.810076+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:04 smithi125 bash[77299]: cluster 2023-10-09T06:18:03.810190+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:04.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:04.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:04.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:18:05.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:05.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:05.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.797041+0000 mon.a (mon.0) 504 : cluster 0 osdmap e110: 8 total, 8 up, 7 in 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.800961+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.801395+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.803083+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.804989+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.805160+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.805736+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.807104+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.810076+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:04 smithi155 bash[59495]: cluster 2023-10-09T06:18:03.810190+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e110 with expected crc 2023-10-09T06:18:05.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:05.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:05.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:05.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.555 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:05.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:05.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:05.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:05.819 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T06:18:05.825 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:18:05.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:05.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:05.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:18:05.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:05.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:06.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.660676+0000 mgr.y (mgr.25723) 383 : cluster 0 pgmap v322: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail; 809 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.788084+0000 mon.a (mon.0) 505 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.804785+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.807415+0000 mon.a (mon.0) 506 : cluster 0 osdmap e111: 8 total, 8 up, 7 in 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.812489+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.075 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.812507+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.813633+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.814247+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: cluster 2023-10-09T06:18:04.816333+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: audit 2023-10-09T06:18:05.170290+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.125:0/1577221811' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: audit 2023-10-09T06:18:05.170703+0000 mon.a (mon.0) 507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: audit 2023-10-09T06:18:05.263090+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.125:0/1470776781' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[80002]: audit 2023-10-09T06:18:05.263660+0000 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.660676+0000 mgr.y (mgr.25723) 383 : cluster 0 pgmap v322: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail; 809 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.788084+0000 mon.a (mon.0) 505 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:18:06.076 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.804785+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.807415+0000 mon.a (mon.0) 506 : cluster 0 osdmap e111: 8 total, 8 up, 7 in 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.812489+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.812507+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.813633+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.814247+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: cluster 2023-10-09T06:18:04.816333+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: audit 2023-10-09T06:18:05.170290+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.125:0/1577221811' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: audit 2023-10-09T06:18:05.170703+0000 mon.a (mon.0) 507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: audit 2023-10-09T06:18:05.263090+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.125:0/1470776781' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.078 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:05 smithi125 bash[77299]: audit 2023-10-09T06:18:05.263660+0000 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.660676+0000 mgr.y (mgr.25723) 383 : cluster 0 pgmap v322: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1014 MiB used, 625 GiB / 626 GiB avail; 809 B/s, 1 keys/s, 4 objects/s recovering 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.788084+0000 mon.a (mon.0) 505 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.804785+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.807415+0000 mon.a (mon.0) 506 : cluster 0 osdmap e111: 8 total, 8 up, 7 in 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.812489+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.812507+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.813633+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.814247+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: cluster 2023-10-09T06:18:04.816333+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:06.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: audit 2023-10-09T06:18:05.170290+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.125:0/1577221811' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: audit 2023-10-09T06:18:05.170703+0000 mon.a (mon.0) 507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: audit 2023-10-09T06:18:05.263090+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.125:0/1470776781' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:05 smithi155 bash[59495]: audit 2023-10-09T06:18:05.263660+0000 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:06.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:06.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:06.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:06.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:06.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:06.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:06.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:06.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:06.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:06.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:06.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:06.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:06.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:06.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:06.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:07.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:07.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:07.098 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.455743+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:07.098 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.797671+0000 mon.a (mon.0) 509 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:07.098 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.805390+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.098 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: audit 2023-10-09T06:18:05.807010+0000 mon.a (mon.0) 510 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:07.098 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: audit 2023-10-09T06:18:05.807107+0000 mon.a (mon.0) 511 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.807172+0000 mon.a (mon.0) 512 : cluster 0 osdmap e112: 8 total, 8 up, 7 in 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.811507+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.813286+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.813405+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: cluster 2023-10-09T06:18:05.816517+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[77299]: audit 2023-10-09T06:18:06.538579+0000 mon.a (mon.0) 513 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.455743+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:07.099 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.797671+0000 mon.a (mon.0) 509 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.805390+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: audit 2023-10-09T06:18:05.807010+0000 mon.a (mon.0) 510 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: audit 2023-10-09T06:18:05.807107+0000 mon.a (mon.0) 511 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.807172+0000 mon.a (mon.0) 512 : cluster 0 osdmap e112: 8 total, 8 up, 7 in 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.811507+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.813286+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.813405+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: cluster 2023-10-09T06:18:05.816517+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.100 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:06 smithi125 bash[80002]: audit 2023-10-09T06:18:06.538579+0000 mon.a (mon.0) 513 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:07.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:07.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.455743+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e111 with expected crc 2023-10-09T06:18:07.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.797671+0000 mon.a (mon.0) 509 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:07.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.805390+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: audit 2023-10-09T06:18:05.807010+0000 mon.a (mon.0) 510 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:07.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: audit 2023-10-09T06:18:05.807107+0000 mon.a (mon.0) 511 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.807172+0000 mon.a (mon.0) 512 : cluster 0 osdmap e112: 8 total, 8 up, 7 in 2023-10-09T06:18:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.811507+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.813286+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.813405+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: cluster 2023-10-09T06:18:05.816517+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-10-09T06:18:07.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:06 smithi155 bash[59495]: audit 2023-10-09T06:18:06.538579+0000 mon.a (mon.0) 513 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:07.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:07.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:07.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:07.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:07.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:07.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:07.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:07.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:07.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:08.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:08.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:08.114 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T06:18:08.119 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:07 smithi125 bash[77299]: cluster 2023-10-09T06:18:06.661195+0000 mgr.y (mgr.25723) 384 : cluster 0 pgmap v325: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:18:08.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:07 smithi125 bash[80002]: cluster 2023-10-09T06:18:06.661195+0000 mgr.y (mgr.25723) 384 : cluster 0 pgmap v325: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:18:08.121 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:18:08.121 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:18:08.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:08.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:07 smithi155 bash[59495]: cluster 2023-10-09T06:18:06.661195+0000 mgr.y (mgr.25723) 384 : cluster 0 pgmap v325: 89 pgs: 6 peering, 2 active+undersized+remapped, 1 unknown, 80 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T06:18:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:08.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:18:08.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:18:08.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_106927 2023-10-09T06:18:08.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:08.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:18:08.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:08.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:08.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:08.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:08.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:08.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:08.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:08.670 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:08.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:08.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:08.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:08.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:08.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:08.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:09.073 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:09.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:18:09.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:09.126 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:09.127 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:09.127 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T06:18:09.127 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:18:09.127 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:18:09.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T06:18:09.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2928 2912 181.991 182 0.00366367 0.00545697 2023-10-09T06:18:09.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:09.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:09.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:09.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:09.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:09.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:09.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:09.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:09.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:09 smithi125 bash[77299]: cluster 2023-10-09T06:18:08.661751+0000 mgr.y (mgr.25723) 385 : cluster 0 pgmap v326: 89 pgs: 2 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 99 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T06:18:09.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:09.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:09 smithi125 bash[80002]: cluster 2023-10-09T06:18:08.661751+0000 mgr.y (mgr.25723) 385 : cluster 0 pgmap v326: 89 pgs: 2 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 99 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T06:18:09.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:09.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:09.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:09.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:09 smithi155 bash[59495]: cluster 2023-10-09T06:18:08.661751+0000 mgr.y (mgr.25723) 385 : cluster 0 pgmap v326: 89 pgs: 2 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 1015 MiB used, 625 GiB / 626 GiB avail; 99 KiB/s, 1 keys/s, 11 objects/s recovering 2023-10-09T06:18:09.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:09.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:10.007 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:10.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:10.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:10.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5536 5520 172.487 163 0.0102271 0.0057859 2023-10-09T06:18:10.168 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:10.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:10.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:10.407 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:10.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:10.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:10.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:10.616 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:10 smithi155 bash[59495]: cluster 2023-10-09T06:18:09.593968+0000 mon.a (mon.0) 514 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:10.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:10.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:10.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:10.796 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:10.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:10.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:10 smithi125 bash[80002]: cluster 2023-10-09T06:18:09.593968+0000 mon.a (mon.0) 514 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:10.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:10 smithi125 bash[77299]: cluster 2023-10-09T06:18:09.593968+0000 mon.a (mon.0) 514 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:10.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:10.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:10.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:10.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:10.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:11.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:11.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:11.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8053 8037 167.419 157.312 0.00741669 0.00596113 2023-10-09T06:18:11.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.244 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:11.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:11.357 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:11] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T06:18:11.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:11.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:11.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:11.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:11.625 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:11 smithi125 bash[77299]: cluster 2023-10-09T06:18:10.662547+0000 mgr.y (mgr.25723) 386 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 91 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 13 MiB/s wr, 201 op/s; 85 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T06:18:11.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:11.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:11.876 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:11.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:11 smithi125 bash[80002]: cluster 2023-10-09T06:18:10.662547+0000 mgr.y (mgr.25723) 386 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 91 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 13 MiB/s wr, 201 op/s; 85 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T06:18:11.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:11 smithi155 bash[59495]: cluster 2023-10-09T06:18:10.662547+0000 mgr.y (mgr.25723) 386 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 91 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 13 MiB/s wr, 201 op/s; 85 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T06:18:11.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:11.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:12.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:12.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 10027 10011 156.405 123.375 0.00392868 0.00638422 2023-10-09T06:18:12.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:12.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:12.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:12.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:12.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:12.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:12.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:12.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:12.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:12.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:12.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:12.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:13.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:13.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:13.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:13.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 12229 12213 152.647 137.625 0.00294215 0.00653609 2023-10-09T06:18:13.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:13.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:13.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[77299]: audit 2023-10-09T06:18:12.196369+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[77299]: audit 2023-10-09T06:18:12.205247+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[77299]: cluster 2023-10-09T06:18:12.663176+0000 mgr.y (mgr.25723) 387 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 328 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 72 KiB/s, 4 objects/s recovering 2023-10-09T06:18:13.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[77299]: audit 2023-10-09T06:18:12.892902+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.402 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[77299]: audit 2023-10-09T06:18:12.904835+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.403 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[80002]: audit 2023-10-09T06:18:12.196369+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.403 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[80002]: audit 2023-10-09T06:18:12.205247+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.403 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[80002]: cluster 2023-10-09T06:18:12.663176+0000 mgr.y (mgr.25723) 387 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 328 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 72 KiB/s, 4 objects/s recovering 2023-10-09T06:18:13.403 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[80002]: audit 2023-10-09T06:18:12.892902+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.403 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:13 smithi125 bash[80002]: audit 2023-10-09T06:18:12.904835+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:13.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:13.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:13 smithi155 bash[59495]: audit 2023-10-09T06:18:12.196369+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:13 smithi155 bash[59495]: audit 2023-10-09T06:18:12.205247+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:13 smithi155 bash[59495]: cluster 2023-10-09T06:18:12.663176+0000 mgr.y (mgr.25723) 387 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 328 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 40 MiB/s wr, 646 op/s; 72 KiB/s, 4 objects/s recovering 2023-10-09T06:18:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:13 smithi155 bash[59495]: audit 2023-10-09T06:18:12.892902+0000 mon.a (mon.0) 517 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:13 smithi155 bash[59495]: audit 2023-10-09T06:18:12.904835+0000 mon.a (mon.0) 518 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:13.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:13.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:13.859 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:13.865 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:13.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:13.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:13.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:14.043 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:14.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:14.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 14560 14544 151.485 145.688 0.0110379 0.0065919 2023-10-09T06:18:14.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:14.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:14.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[77299]: audit 2023-10-09T06:18:13.524367+0000 mon.a (mon.0) 519 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:14.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[77299]: audit 2023-10-09T06:18:13.525740+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:14.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:14.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.444 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:14 smithi155 bash[59495]: audit 2023-10-09T06:18:13.524367+0000 mon.a (mon.0) 519 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:14.444 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:14 smithi155 bash[59495]: audit 2023-10-09T06:18:13.525740+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:14.444 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:14 smithi155 bash[59495]: audit 2023-10-09T06:18:13.535677+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:14.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:14.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:14.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[77299]: audit 2023-10-09T06:18:13.535677+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[80002]: audit 2023-10-09T06:18:13.524367+0000 mon.a (mon.0) 519 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[80002]: audit 2023-10-09T06:18:13.525740+0000 mon.a (mon.0) 520 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:14.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:14 smithi125 bash[80002]: audit 2023-10-09T06:18:13.535677+0000 mon.a (mon.0) 521 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:14.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:14.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:14.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:14.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:14.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:14.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:14.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:14.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:14.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:14.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:15.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:15.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:15.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 16020 16004 142.879 91.25 0.00676552 0.00699217 2023-10-09T06:18:15.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:15.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:15.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:15.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:15.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:15.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:15.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:15 smithi155 bash[59495]: cluster 2023-10-09T06:18:14.663856+0000 mgr.y (mgr.25723) 388 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 687 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 KiB/s, 3 objects/s recovering 2023-10-09T06:18:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:15.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:15.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:15.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:15 smithi125 bash[77299]: cluster 2023-10-09T06:18:14.663856+0000 mgr.y (mgr.25723) 388 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 687 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 KiB/s, 3 objects/s recovering 2023-10-09T06:18:15.628 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:15 smithi125 bash[80002]: cluster 2023-10-09T06:18:14.663856+0000 mgr.y (mgr.25723) 388 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 687 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 KiB/s, 3 objects/s recovering 2023-10-09T06:18:15.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:15.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:15.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:15.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:15.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:15.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:15.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:16.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:16.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:16.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:16.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 18115 18099 141.385 130.938 0.0082457 0.00706581 2023-10-09T06:18:16.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:16.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:16.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:16.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:16.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:16.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:16.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:16.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:16.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:16.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:16.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:16.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:17.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:17.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 20159 20143 139.868 127.75 0.0076894 0.00714286 2023-10-09T06:18:17.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:17.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:17.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:17.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:17.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:17.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:17.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:17.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:17.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:17.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:17.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:17.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:17 smithi155 bash[59495]: cluster 2023-10-09T06:18:16.664429+0000 mgr.y (mgr.25723) 389 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1004 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:18:17.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:17.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:17.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:18.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.102 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:17 smithi125 bash[77299]: cluster 2023-10-09T06:18:16.664429+0000 mgr.y (mgr.25723) 389 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1004 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:18:18.102 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:17 smithi125 bash[80002]: cluster 2023-10-09T06:18:16.664429+0000 mgr.y (mgr.25723) 389 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 1004 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 53 KiB/s, 3 objects/s recovering 2023-10-09T06:18:18.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:18.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 22219 22203 138.755 128.75 0.00327092 0.0072004 2023-10-09T06:18:18.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:18.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:18.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:18.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:18.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:18.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:18.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:18.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:18.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:18.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:18.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:18 smithi155 bash[59495]: audit 2023-10-09T06:18:18.527285+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:18 smithi155 bash[59495]: audit 2023-10-09T06:18:18.528471+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:18 smithi155 bash[59495]: audit 2023-10-09T06:18:18.529170+0000 mon.a (mon.0) 524 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-09T06:18:18.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:18 smithi155 bash[59495]: audit 2023-10-09T06:18:18.562402+0000 mon.a (mon.0) 525 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:18.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[77299]: audit 2023-10-09T06:18:18.527285+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.903 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[77299]: audit 2023-10-09T06:18:18.528471+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[77299]: audit 2023-10-09T06:18:18.529170+0000 mon.a (mon.0) 524 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[77299]: audit 2023-10-09T06:18:18.562402+0000 mon.a (mon.0) 525 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[80002]: audit 2023-10-09T06:18:18.527285+0000 mon.a (mon.0) 522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[80002]: audit 2023-10-09T06:18:18.528471+0000 mon.a (mon.0) 523 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[80002]: audit 2023-10-09T06:18:18.529170+0000 mon.a (mon.0) 524 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-09T06:18:18.904 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:18 smithi125 bash[80002]: audit 2023-10-09T06:18:18.562402+0000 mon.a (mon.0) 525 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:18.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:18.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:19.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:19.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:19.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 23932 23916 135.873 107.062 0.00528082 0.00735135 2023-10-09T06:18:19.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:19.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:19.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:19.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:19.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:19.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:19.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:19.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:19.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:19.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:19.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:19.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: cluster 2023-10-09T06:18:18.664981+0000 mgr.y (mgr.25723) 390 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 48 KiB/s, 0 objects/s recovering 2023-10-09T06:18:19.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: audit 2023-10-09T06:18:18.734020+0000 mon.a (mon.0) 526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]': finished 2023-10-09T06:18:19.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: audit 2023-10-09T06:18:18.734102+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2023-10-09T06:18:19.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: audit 2023-10-09T06:18:18.734173+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-09T06:18:19.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: cluster 2023-10-09T06:18:18.734223+0000 mon.a (mon.0) 529 : cluster 0 osdmap e113: 8 total, 8 up, 7 in 2023-10-09T06:18:19.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: cluster 2023-10-09T06:18:18.736085+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: cluster 2023-10-09T06:18:18.747812+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: cluster 2023-10-09T06:18:18.749457+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[77299]: audit 2023-10-09T06:18:18.776021+0000 mon.a (mon.0) 530 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: cluster 2023-10-09T06:18:18.664981+0000 mgr.y (mgr.25723) 390 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 48 KiB/s, 0 objects/s recovering 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: audit 2023-10-09T06:18:18.734020+0000 mon.a (mon.0) 526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]': finished 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: audit 2023-10-09T06:18:18.734102+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: audit 2023-10-09T06:18:18.734173+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: cluster 2023-10-09T06:18:18.734223+0000 mon.a (mon.0) 529 : cluster 0 osdmap e113: 8 total, 8 up, 7 in 2023-10-09T06:18:19.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: cluster 2023-10-09T06:18:18.736085+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: cluster 2023-10-09T06:18:18.747812+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: cluster 2023-10-09T06:18:18.749457+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:19.992 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:19 smithi125 bash[80002]: audit 2023-10-09T06:18:18.776021+0000 mon.a (mon.0) 530 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:20.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:20.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:20.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:20.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:20.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: cluster 2023-10-09T06:18:18.664981+0000 mgr.y (mgr.25723) 390 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 48 KiB/s, 0 objects/s recovering 2023-10-09T06:18:20.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: audit 2023-10-09T06:18:18.734020+0000 mon.a (mon.0) 526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [1, 2]}]': finished 2023-10-09T06:18:20.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: audit 2023-10-09T06:18:18.734102+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: audit 2023-10-09T06:18:18.734173+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: cluster 2023-10-09T06:18:18.734223+0000 mon.a (mon.0) 529 : cluster 0 osdmap e113: 8 total, 8 up, 7 in 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: cluster 2023-10-09T06:18:18.736085+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: cluster 2023-10-09T06:18:18.747812+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: cluster 2023-10-09T06:18:18.749457+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e113 with expected crc 2023-10-09T06:18:20.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:19 smithi155 bash[59495]: audit 2023-10-09T06:18:18.776021+0000 mon.a (mon.0) 530 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:18:20.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 25883 25867 134.711 121.938 0.00558304 0.00741821 2023-10-09T06:18:20.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:20.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:20.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:20.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:20.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:20.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:20.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:20.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:20.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[77299]: cluster 2023-10-09T06:18:19.739063+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[77299]: cluster 2023-10-09T06:18:19.739161+0000 mon.a (mon.0) 531 : cluster 0 osdmap e114: 8 total, 8 up, 7 in 2023-10-09T06:18:20.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[77299]: cluster 2023-10-09T06:18:19.750357+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.776 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[77299]: cluster 2023-10-09T06:18:19.753503+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.777 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[77299]: cluster 2023-10-09T06:18:19.756268+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.777 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[80002]: cluster 2023-10-09T06:18:19.739063+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.777 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[80002]: cluster 2023-10-09T06:18:19.739161+0000 mon.a (mon.0) 531 : cluster 0 osdmap e114: 8 total, 8 up, 7 in 2023-10-09T06:18:20.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[80002]: cluster 2023-10-09T06:18:19.750357+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[80002]: cluster 2023-10-09T06:18:19.753503+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:20 smithi125 bash[80002]: cluster 2023-10-09T06:18:19.756268+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:20.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:20 smithi155 bash[59495]: cluster 2023-10-09T06:18:19.739063+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:20 smithi155 bash[59495]: cluster 2023-10-09T06:18:19.739161+0000 mon.a (mon.0) 531 : cluster 0 osdmap e114: 8 total, 8 up, 7 in 2023-10-09T06:18:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:20 smithi155 bash[59495]: cluster 2023-10-09T06:18:19.750357+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:20 smithi155 bash[59495]: cluster 2023-10-09T06:18:19.753503+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:20 smithi155 bash[59495]: cluster 2023-10-09T06:18:19.756268+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-09T06:18:20.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:20.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:20.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:21.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:21.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:21.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 27895 27879 134.021 125.75 0.015259 0.00745526 2023-10-09T06:18:21.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:21.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.303 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:21] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:18:21.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:21.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:21.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:21.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:21.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:21.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:21.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:21.812 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:21.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:21.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:21 smithi125 bash[77299]: cluster 2023-10-09T06:18:20.665414+0000 mgr.y (mgr.25723) 391 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T06:18:21.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:21 smithi125 bash[80002]: cluster 2023-10-09T06:18:20.665414+0000 mgr.y (mgr.25723) 391 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T06:18:21.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:21.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:21.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:22.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:22.090 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:21 smithi155 bash[59495]: cluster 2023-10-09T06:18:20.665414+0000 mgr.y (mgr.25723) 391 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-09T06:18:22.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:22.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:22.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 29796 29780 132.934 118.812 0.00808257 0.00751595 2023-10-09T06:18:22.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:22.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:22.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:22.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:22.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:22.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:22.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:22.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:22.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:22.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:22.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:22.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:22.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:22.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:23.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:23.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:23.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:23.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 31470 31454 131.045 104.625 0.0194154 0.00762317 2023-10-09T06:18:23.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:23.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:23.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:23.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:23.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:23.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:23.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:23.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:23.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:23.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:23.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:23.933 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:23 smithi155 bash[59495]: cluster 2023-10-09T06:18:22.666081+0000 mgr.y (mgr.25723) 392 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.6 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:18:23.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:23.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:24.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:24.118 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:23 smithi125 bash[77299]: cluster 2023-10-09T06:18:22.666081+0000 mgr.y (mgr.25723) 392 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.6 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:18:24.118 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:23 smithi125 bash[80002]: cluster 2023-10-09T06:18:22.666081+0000 mgr.y (mgr.25723) 392 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.6 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-10-09T06:18:24.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 32859 32843 128.27 86.8125 0.00378782 0.00779044 2023-10-09T06:18:24.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:24.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:24.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:24.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:24.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:24.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:24.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:24.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:24.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:24.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:24.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:25.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:25.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:25.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:25.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 34528 34512 126.86 104.312 0.00596652 0.00787565 2023-10-09T06:18:25.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:25.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:25.367 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:25.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:25.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:25.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:25.767 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:25 smithi125 bash[77299]: cluster 2023-10-09T06:18:24.666666+0000 mgr.y (mgr.25723) 393 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:18:25.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:25 smithi125 bash[80002]: cluster 2023-10-09T06:18:24.666666+0000 mgr.y (mgr.25723) 393 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:18:25.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:25.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:25.918 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:25 smithi155 bash[59495]: cluster 2023-10-09T06:18:24.666666+0000 mgr.y (mgr.25723) 393 : cluster 0 pgmap v336: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:18:25.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:25.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:26.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:26.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:26.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:26.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 36286 36270 125.916 109.875 0.00334639 0.00793707 2023-10-09T06:18:26.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:26.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:26.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:26.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:26.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:26.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:26.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:26.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:26.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:26.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:26.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:27.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:27.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:27.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:27.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 37900 37884 124.597 100.875 0.00425184 0.00801941 2023-10-09T06:18:27.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:27.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:27.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:27.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:27.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:27.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:27.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:27.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:27 smithi155 bash[59495]: cluster 2023-10-09T06:18:26.667238+0000 mgr.y (mgr.25723) 394 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:18:27.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:27.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:28.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:28.045 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:27 smithi125 bash[77299]: cluster 2023-10-09T06:18:26.667238+0000 mgr.y (mgr.25723) 394 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:18:28.045 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:27 smithi125 bash[80002]: cluster 2023-10-09T06:18:26.667238+0000 mgr.y (mgr.25723) 394 : cluster 0 pgmap v337: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:18:28.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:28.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:18:28.165581+0000 min lat: 0.00139442 max lat: 0.335425 avg lat: 0.00811887 2023-10-09T06:18:28.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:28.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 39404 39388 123.067 94 0.0124461 0.00811887 2023-10-09T06:18:28.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:28.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:28.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:28.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:28.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:28.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:28.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:28.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:28.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:28.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:28.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:28.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:28.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:28.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:28.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:28.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:28.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:29.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:29.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:29.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:29.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 40794 40778 121.344 86.875 0.0179034 0.00823575 2023-10-09T06:18:29.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:29.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:29.322 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:17586 5.3 5:c120002a:::benchmark_data_smithi125_106927_object17585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:15.844654+0000", 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.463937961999999, 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.094644768000000004, 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.323 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 17586 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844654+0000", 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844654+0000", 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844653+0000", 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844692+0000", 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9026999999999998e-05 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844715+0000", 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.283e-05 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.844722+0000", 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2899999999999999e-06 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.846456+0000", 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017346359999999999 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.847644+0000", 2023-10-09T06:18:29.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001188035 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.847801+0000", 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015670199999999999 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850466+0000", 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026650979999999999 2023-10-09T06:18:29.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.938967+0000", 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.088500793999999994 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939074+0000", 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107068 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939205+0000", 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000131366 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939299+0000", 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3980999999999993e-05 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:17600 5.3 5:c0593339:::benchmark_data_smithi125_106927_object17599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:15.849517+0000", 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.459074960000001, 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.092960005999999998, 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.331 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 17600 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849517+0000", 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849517+0000", 2023-10-09T06:18:29.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849516+0000", 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849541+0000", 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5205e-05 2023-10-09T06:18:29.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849551+0000", 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.363e-06 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.849557+0000", 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0970000000000002e-06 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850686+0000", 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011294370000000001 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850837+0000", 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015113600000000001 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:29.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936678+0000", 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085840527999999999 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.942315+0000", 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005637497 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.942383+0000", 2023-10-09T06:18:29.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8015999999999994e-05 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.942397+0000", 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.349e-05 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.942442+0000", 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4807e-05 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.942477+0000", 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5818999999999997e-05 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:17598 5.c e112/105 5:30a23aa8:::benchmark_data_smithi125_106927_object17597:head v 112'1107, mlcod=112'1104)", 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:15.850611+0000", 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.457981608000001, 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085837537000000005, 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850611+0000", 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850611+0000", 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850610+0000", 2023-10-09T06:18:29.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850639+0000", 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8938999999999999e-05 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850649+0000", 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5400000000000001e-06 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850656+0000", 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4120000000000004e-06 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850727+0000", 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0588999999999998e-05 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.850745+0000", 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8314000000000001e-05 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936349+0000", 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.085603290999999998 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936448+0000", 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9821999999999994e-05 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:17603 5.e e112/105 5:74ce6e77:::benchmark_data_smithi125_106927_object17602:head v 112'1157, mlcod=112'1155)", 2023-10-09T06:18:29.346 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:15.851891+0000", 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.456701156999999, 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.087583521999999997, 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.851891+0000", 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.851891+0000", 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.851891+0000", 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.851962+0000", 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1681999999999996e-05 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936331+0000", 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.084369221999999994 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936368+0000", 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.697e-05 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936484+0000", 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000115766 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936526+0000", 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1561999999999999e-05 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939420+0000", 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028941779999999999 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939475+0000", 2023-10-09T06:18:29.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4897e-05 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:17604 5.3 5:c2a285d6:::benchmark_data_smithi125_106927_object17603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:15.853061+0000", 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 13.45553125, 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.093850330999999995, 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.352 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 17604 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.853061+0000", 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.853061+0000", 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.853060+0000", 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.853087+0000", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7217e-05 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936347+0000", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.083259811000000003 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.936370+0000", 2023-10-09T06:18:29.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3297000000000001e-05 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939371+0000", 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003000628 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.939679+0000", 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030776799999999998 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.940159+0000", 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048050799999999999 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.946577+0000", 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006418272 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.946587+0000", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7189999999999996e-06 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.946714+0000", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012670199999999999 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.946785+0000", 2023-10-09T06:18:29.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1500000000000003e-05 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.946912+0000", 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126279 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.359 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:19396 5.5 5:ae5b6f40:::benchmark_data_smithi125_106927_object19395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.744566+0000", 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.564025998, 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.083055384999999995, 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 19396 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.360 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744566+0000", 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744566+0000", 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744565+0000", 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744686+0000", 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012157 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744697+0000", 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0609e-05 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744704+0000", 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3780000000000003e-06 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744798+0000", 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3231000000000002e-05 2023-10-09T06:18:29.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744918+0000", 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120767 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745049+0000", 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013015100000000001 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746816+0000", 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001767656 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.826055+0000", 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.079239292000000003 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.826087+0000", 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1656000000000003e-05 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.827575+0000", 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001487464 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.827622+0000", 2023-10-09T06:18:29.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7221000000000001e-05 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19391 5.e e112/105 5:789e07c0:::benchmark_data_smithi125_106927_object19390:head v 112'1269, mlcod=112'1267)", 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.744646+0000", 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.563946303, 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.081221957999999997, 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.367 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744646+0000", 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744646+0000", 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744646+0000", 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744686+0000", 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0701000000000002e-05 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744696+0000", 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6600000000000007e-06 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744703+0000", 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4799999999999998e-06 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744801+0000", 2023-10-09T06:18:29.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8430000000000005e-05 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.744824+0000", 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2705e-05 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825824+0000", 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.081000600000000006 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825868+0000", 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3761000000000003e-05 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19393 5.7 e112/105 5:e22b8145:::benchmark_data_smithi125_106927_object19392:head v 112'1241, mlcod=112'1238)", 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.745778+0000", 2023-10-09T06:18:29.372 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.562814618999999, 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080054155000000002, 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745778+0000", 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745778+0000", 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745777+0000", 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745901+0000", 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012401400000000001 2023-10-09T06:18:29.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745913+0000", 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1440000000000001e-05 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745920+0000", 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6279999999999996e-06 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745979+0000", 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8400000000000003e-05 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.745997+0000", 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8614000000000002e-05 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748098+0000", 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021010859999999998 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825832+0000", 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.077733567000000003 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.377 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19398 5.2 e112/105 5:4bc34fa9:::benchmark_data_smithi125_106927_object19397:head v 112'1173, mlcod=112'1170)", 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.746567+0000", 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.562025500000001, 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.079297872000000005, 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746567+0000", 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746567+0000", 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746566+0000", 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746611+0000", 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5157000000000002e-05 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746622+0000", 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0633e-05 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746628+0000", 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4459999999999998e-06 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747146+0000", 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051749599999999999 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747166+0000", 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0536000000000001e-05 2023-10-09T06:18:29.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.749223+0000", 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020571909999999999 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825865+0000", 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.076641332000000006 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.382 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19392 5.8 e112/105 5:1f0bc4f5:::benchmark_data_smithi125_106927_object19391:head v 112'1150, mlcod=112'1148)", 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.746658+0000", 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.561934188, 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.079177476999999996, 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746658+0000", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746658+0000", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746658+0000", 2023-10-09T06:18:29.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746684+0000", 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6313999999999999e-05 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746694+0000", 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6390000000000004e-06 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746700+0000", 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1770000000000003e-06 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746748+0000", 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.816e-05 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.746771+0000", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3229000000000001e-05 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825800+0000", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.079028661 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825836+0000", 2023-10-09T06:18:29.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5864000000000001e-05 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19400 5.c e112/105 5:3af96a6d:::benchmark_data_smithi125_106927_object19399:head v 112'1203, mlcod=112'1201)", 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.747004+0000", 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.561588256, 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078869515000000001, 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.388 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747004+0000", 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747004+0000", 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747004+0000", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747039+0000", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4950999999999998e-05 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747050+0000", 2023-10-09T06:18:29.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1518999999999999e-05 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747057+0000", 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3000000000000004e-06 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747100+0000", 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2432999999999998e-05 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747119+0000", 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8819999999999999e-05 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825791+0000", 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078672727999999997 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825874+0000", 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2317999999999993e-05 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19399 5.d e112/108 5:ba9950c9:::benchmark_data_smithi125_106927_object19398:head v 112'1240, mlcod=112'1238)", 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.747525+0000", 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.561067142000001, 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.081280080000000005, 2023-10-09T06:18:29.393 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747525+0000", 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747525+0000", 2023-10-09T06:18:29.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747516+0000", 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999905 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747554+0000", 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8334000000000003e-05 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747563+0000", 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2949999999999999e-06 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.747570+0000", 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2349999999999996e-06 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825822+0000", 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078252499000000003 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825853+0000", 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0969000000000001e-05 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.828747+0000", 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028940239999999998 2023-10-09T06:18:29.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.828805+0000", 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8177000000000002e-05 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19403 5.7 e112/105 5:e0467ce2:::benchmark_data_smithi125_106927_object19402:head v 112'1242, mlcod=112'1239)", 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.748171+0000", 2023-10-09T06:18:29.398 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.560421401999999, 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080710682000000006, 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748171+0000", 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748171+0000", 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748170+0000", 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.749028+0000", 2023-10-09T06:18:29.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00085783699999999997 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825770+0000", 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.076741895000000004 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825795+0000", 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5286000000000002e-05 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825939+0000", 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014387199999999999 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825963+0000", 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3597000000000001e-05 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.828761+0000", 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027978769999999998 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.828882+0000", 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121097 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:19402 5.b e112/105 5:d237e9c7:::benchmark_data_smithi125_106927_object19401:head v 112'1212, mlcod=112'1210)", 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:16.748239+0000", 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 12.560353613, 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.083516471999999994, 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748239+0000", 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748239+0000", 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748238+0000", 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.748277+0000", 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9035999999999997e-05 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825786+0000", 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.077508545999999998 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825801+0000", 2023-10-09T06:18:29.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5267000000000001e-05 2023-10-09T06:18:29.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825903+0000", 2023-10-09T06:18:29.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010174399999999999 2023-10-09T06:18:29.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.825926+0000", 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2952e-05 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.831723+0000", 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0057973900000000004 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:16.831755+0000", 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2017999999999997e-05 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:31588 5.3 5:ce030f5c:::benchmark_data_smithi125_106927_object31587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:23.254534+0000", 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 6.0540588839999998, 2023-10-09T06:18:29.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.078132942999999996, 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 31588 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254534+0000", 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254534+0000", 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254533+0000", 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254567+0000", 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3991e-05 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254578+0000", 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1527e-05 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.254586+0000", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7209999999999994e-06 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.255279+0000", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00069339800000000002 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.255398+0000", 2023-10-09T06:18:29.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119082 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.255531+0000", 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013247899999999999 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.256299+0000", 2023-10-09T06:18:29.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00076794099999999996 2023-10-09T06:18:29.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.331840+0000", 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.075541128999999999 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.332580+0000", 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00073981199999999996 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.332629+0000", 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9379000000000003e-05 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.332667+0000", 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7388999999999997e-05 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.420 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:32597 5.5 5:a30b845d:::benchmark_data_smithi125_106927_object32596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:23.901334+0000", 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 5.4072585789999996, 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.129955442, 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.424 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 32597 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901334+0000", 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901334+0000", 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901333+0000", 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901352+0000", 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8425000000000001e-05 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901358+0000", 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5919999999999997e-06 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901363+0000", 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0590000000000002e-06 2023-10-09T06:18:29.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.902909+0000", 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001545392 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.903027+0000", 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118156 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.903168+0000", 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140884 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.905775+0000", 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026071660000000002 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.429 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.911968+0000", 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061929680000000001 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.031203+0000", 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.11923511000000001 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.031253+0000", 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9552999999999998e-05 2023-10-09T06:18:29.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.031289+0000", 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6637000000000002e-05 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:32600 5.5 5:a42be9fe:::benchmark_data_smithi125_106927_object32599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.431 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:23.901368+0000", 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 5.4072243159999998, 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.132838284, 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 32600 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901368+0000", 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901368+0000", 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901368+0000", 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901390+0000", 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2838999999999999e-05 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901396+0000", 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9089999999999997e-06 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.901402+0000", 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7049999999999996e-06 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.904234+0000", 2023-10-09T06:18:29.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028322159999999998 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.904368+0000", 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013385200000000001 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.904506+0000", 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013786000000000001 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.906032+0000", 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015258800000000001 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.912003+0000", 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0059706179999999996 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034114+0000", 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.122110967 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034166+0000", 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2207999999999999e-05 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.438 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034206+0000", 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0707000000000002e-05 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:32611 5.5 5:a4405ea6:::benchmark_data_smithi125_106927_object32610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:23.908616+0000", 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 5.399976197, 2023-10-09T06:18:29.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12574507900000001, 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 32611 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908616+0000", 2023-10-09T06:18:29.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908616+0000", 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908615+0000", 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:29.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908646+0000", 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1285999999999998e-05 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908660+0000", 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3448000000000001e-05 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.908667+0000", 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0949999999999998e-06 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.909006+0000", 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033913599999999997 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.909096+0000", 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0019000000000007e-05 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.909189+0000", 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2689000000000007e-05 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.912010+0000", 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028208619999999999 2023-10-09T06:18:29.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.912371+0000", 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00036146000000000003 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034252+0000", 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.121881354 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034326+0000", 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3403000000000002e-05 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.034361+0000", 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5553000000000003e-05 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.446 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:32615 5.5 5:a30af8cc:::benchmark_data_smithi125_106927_object32614:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:23.910145+0000", 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 5.3984475649999997, 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12503418499999999, 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 32615 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910145+0000", 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910145+0000", 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910144+0000", 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910167+0000", 2023-10-09T06:18:29.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2569999999999999e-05 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910179+0000", 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2029e-05 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910187+0000", 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0059999999999992e-06 2023-10-09T06:18:29.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910225+0000", 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8515e-05 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910348+0000", 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122912 2023-10-09T06:18:29.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.910504+0000", 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015620900000000001 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.912076+0000", 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015719149999999999 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:23.912398+0000", 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000321909 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.035033+0000", 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12263545200000001 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.035115+0000", 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1884000000000004e-05 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:24.035179+0000", 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3726999999999995e-05 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:34337 5.5 5:a3f85be2:::benchmark_data_smithi125_106927_object34336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:25.003511+0000", 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 4.3050814959999997, 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.080667210000000003, 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 34337 2023-10-09T06:18:29.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003511+0000", 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003511+0000", 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003510+0000", 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003534+0000", 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4335e-05 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003566+0000", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2325e-05 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.003610+0000", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3158999999999998e-05 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.010028+0000", 2023-10-09T06:18:29.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064179010000000002 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.010159+0000", 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013104600000000001 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.010448+0000", 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028988400000000002 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.013855+0000", 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034068549999999999 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.014134+0000", 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027850500000000002 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.016788+0000", 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002654394 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.022655+0000", 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058663730000000002 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:18:29.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:25.084178+0000", 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.061523545999999998 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:18:29.462 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:18:29.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:29.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:29.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:29.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:29.627 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:18:29.631 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:18:29.672 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:18:29.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:29.734 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:18:29.740 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:29 smithi155 bash[59495]: cluster 2023-10-09T06:18:28.667940+0000 mgr.y (mgr.25723) 395 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:18:29.740 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:29 smithi155 bash[59495]: audit 2023-10-09T06:18:29.113045+0000 mon.a (mon.0) 532 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:29.740 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:29 smithi155 bash[59495]: audit 2023-10-09T06:18:29.576926+0000 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.125:0/429123124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:29.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:29.768 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:18:29.789 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:18:29.822 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:18:29.854 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:18:29.854 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:18:29.854 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:18:29.855 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:18:29.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:29.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[77299]: cluster 2023-10-09T06:18:28.667940+0000 mgr.y (mgr.25723) 395 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:18:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[77299]: audit 2023-10-09T06:18:29.113045+0000 mon.a (mon.0) 532 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:29.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[77299]: audit 2023-10-09T06:18:29.576926+0000 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.125:0/429123124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:29.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[80002]: cluster 2023-10-09T06:18:28.667940+0000 mgr.y (mgr.25723) 395 : cluster 0 pgmap v338: 89 pgs: 89 active+clean; 2.2 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:18:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[80002]: audit 2023-10-09T06:18:29.113045+0000 mon.a (mon.0) 532 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:29 smithi125 bash[80002]: audit 2023-10-09T06:18:29.576926+0000 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.125:0/429123124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:29.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:29.893 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:18:29.900 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.901 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:29.992 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:30.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:30.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 42064 42048 119.436 79.375 0.00935909 0.0083676 2023-10-09T06:18:30.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:30.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:30.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:30.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:30.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:30.550 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:18:30.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.599805+0000 mon.a (mon.0) 534 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: audit 2023-10-09T06:18:29.620077+0000 mon.a (mon.0) 535 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.620117+0000 mon.a (mon.0) 536 : cluster 0 osdmap e115: 8 total, 8 up, 7 in 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.620344+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.621535+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.618 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.623152+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.619 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:30 smithi155 bash[59495]: cluster 2023-10-09T06:18:29.627773+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:30.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:30.816 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.599805+0000 mon.a (mon.0) 534 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: audit 2023-10-09T06:18:29.620077+0000 mon.a (mon.0) 535 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.620117+0000 mon.a (mon.0) 536 : cluster 0 osdmap e115: 8 total, 8 up, 7 in 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.620344+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.621535+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.623152+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[77299]: cluster 2023-10-09T06:18:29.627773+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.817 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.599805+0000 mon.a (mon.0) 534 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: audit 2023-10-09T06:18:29.620077+0000 mon.a (mon.0) 535 : audit 1 from='client.? 172.21.15.125:0/656382036' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.620117+0000 mon.a (mon.0) 536 : cluster 0 osdmap e115: 8 total, 8 up, 7 in 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.620344+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.621535+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.623152+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:30 smithi125 bash[80002]: cluster 2023-10-09T06:18:29.627773+0000 osd.4 (osd.4) 36 : cluster 3 failed to encode map e115 with expected crc 2023-10-09T06:18:30.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:30.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:30.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:30.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:31.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:31.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 43550 43534 118.281 92.875 0.00781146 0.0084489 2023-10-09T06:18:31.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:31.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:31.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:31] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:18:31.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:31.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:31.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:31.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:31.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:31.878 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:31 smithi125 bash[77299]: cluster 2023-10-09T06:18:30.668370+0000 mgr.y (mgr.25723) 396 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T06:18:31.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:31 smithi125 bash[80002]: cluster 2023-10-09T06:18:30.668370+0000 mgr.y (mgr.25723) 396 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T06:18:31.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:31.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:31 smithi155 bash[59495]: cluster 2023-10-09T06:18:30.668370+0000 mgr.y (mgr.25723) 396 : cluster 0 pgmap v340: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T06:18:31.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:32.007 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:32.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:32.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:32.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 45084 45068 117.347 95.875 0.00303216 0.00851638 2023-10-09T06:18:32.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:32.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:32.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:32.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:32.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:32.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:32.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:32.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:32.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:32.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:33.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:33.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:33.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 46505 46489 116.205 88.8125 0.0129046 0.00859963 2023-10-09T06:18:33.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:33.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:33.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:33.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:33.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:33.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:33.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:33.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:33.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.852 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:33 smithi155 bash[59495]: cluster 2023-10-09T06:18:32.668901+0000 mgr.y (mgr.25723) 397 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:18:33.852 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:33 smithi155 bash[59495]: audit 2023-10-09T06:18:33.563858+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:33.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:33.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:33.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:34.002 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:33 smithi125 bash[77299]: cluster 2023-10-09T06:18:32.668901+0000 mgr.y (mgr.25723) 397 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:18:34.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:33 smithi125 bash[77299]: audit 2023-10-09T06:18:33.563858+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:34.003 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:33 smithi125 bash[80002]: cluster 2023-10-09T06:18:32.668901+0000 mgr.y (mgr.25723) 397 : cluster 0 pgmap v341: 89 pgs: 89 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:18:34.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:33 smithi125 bash[80002]: audit 2023-10-09T06:18:33.563858+0000 mon.a (mon.0) 537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:34.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:34.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:34.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:34.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 47876 47860 115.031 85.6875 0.0101768 0.00868767 2023-10-09T06:18:34.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:34.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:34.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:34.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:34.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:34.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:34.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:34.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:34.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:34.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:34.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:34.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:34.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:34.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:34.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:35.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:35.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:35.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 49305 49289 114.078 89.3125 0.00890416 0.00876019 2023-10-09T06:18:35.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:35.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:35.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:35.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:35.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:35.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:35.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:35.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:35.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:35.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:35.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:35.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:35.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:35.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:35 smithi155 bash[59495]: cluster 2023-10-09T06:18:34.669554+0000 mgr.y (mgr.25723) 398 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:18:35.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:35.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:35.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:36.023 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:18:36.033 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:35 smithi125 bash[77299]: cluster 2023-10-09T06:18:34.669554+0000 mgr.y (mgr.25723) 398 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:18:36.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:35 smithi125 bash[80002]: cluster 2023-10-09T06:18:34.669554+0000 mgr.y (mgr.25723) 398 : cluster 0 pgmap v342: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:18:36.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:36.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:36.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:36.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 50650 50634 113.006 84.0625 0.0230238 0.00884088 2023-10-09T06:18:36.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:36.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:36.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:36.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:36.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:36.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:36.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:36.537 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:36.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:36.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:36.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:36.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:36.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:36.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:36.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:37.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:37.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:37.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 52018 52002 112.057 85.5 0.00848791 0.00891756 2023-10-09T06:18:37.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:37.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:37.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:37.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:37.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:37.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:37.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:37.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:37.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:37.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:37.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:37 smithi125 bash[77299]: cluster 2023-10-09T06:18:36.670101+0000 mgr.y (mgr.25723) 399 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:18:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:37 smithi125 bash[80002]: cluster 2023-10-09T06:18:36.670101+0000 mgr.y (mgr.25723) 399 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:18:37.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:37.970 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:37 smithi155 bash[59495]: cluster 2023-10-09T06:18:36.670101+0000 mgr.y (mgr.25723) 399 : cluster 0 pgmap v343: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:18:37.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:37.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:37.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:38.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:38.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:38.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 53395 53379 111.191 86.0625 0.0264761 0.00898836 2023-10-09T06:18:38.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:38.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:38.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:38.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:38.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:38.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:38.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:38.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:38.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:38.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:38.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:38.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:38.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:38.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:39.004 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:39.004 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:39.005 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T06:18:39.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T06:18:39.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:39.100 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:39.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 54660 54644 110.154 79.0625 0.00856619 0.00907264 2023-10-09T06:18:39.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:39.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:39.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:39.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:39.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:39.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:39.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:39.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:39.628 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:39 smithi125 bash[77299]: cluster 2023-10-09T06:18:38.670552+0000 mgr.y (mgr.25723) 400 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:39.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:39.744 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:39.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:39.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:39 smithi125 bash[80002]: cluster 2023-10-09T06:18:38.670552+0000 mgr.y (mgr.25723) 400 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:39.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:39 smithi155 bash[59495]: cluster 2023-10-09T06:18:38.670552+0000 mgr.y (mgr.25723) 400 : cluster 0 pgmap v344: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:39.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:39.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:40.007 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:40.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:40.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:40.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 15 55958 55943 109.249 81.1875 0.00751539 0.00914798 2023-10-09T06:18:40.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:40.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:40.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:40.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:40.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:40.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:40.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:40.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:40.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:40.891 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:40.892 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:40.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:40.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:41.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:41.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:41.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 57254 57238 108.39 80.9375 0.0121742 0.00922077 2023-10-09T06:18:41.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:41.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:41.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:41.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:41] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:18:41.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:41.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:41.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:41.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:41.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:41.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:41 smithi125 bash[77299]: cluster 2023-10-09T06:18:40.671116+0000 mgr.y (mgr.25723) 401 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:18:41.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:41 smithi125 bash[80002]: cluster 2023-10-09T06:18:40.671116+0000 mgr.y (mgr.25723) 401 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:18:41.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:41.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.872 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:41 smithi155 bash[59495]: cluster 2023-10-09T06:18:40.671116+0000 mgr.y (mgr.25723) 401 : cluster 0 pgmap v345: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:18:41.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:41.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:41.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:42.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:42.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 58390 58374 107.291 71 0.00618414 0.00931494 2023-10-09T06:18:42.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:42.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:42.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:42.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:42.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:42.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:42.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:42.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:42.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:42.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:42.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:43.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:43.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 59750 59734 106.654 85 0.00880123 0.00936989 2023-10-09T06:18:43.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:43.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:43.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:43.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:43.527 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:43.527 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:43.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:43.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:43.673 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:43.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:43.734 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:43 smithi155 bash[59495]: cluster 2023-10-09T06:18:42.671673+0000 mgr.y (mgr.25723) 402 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:43.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:43.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:43.869 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:43 smithi125 bash[77299]: cluster 2023-10-09T06:18:42.671673+0000 mgr.y (mgr.25723) 402 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:43.870 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:43 smithi125 bash[80002]: cluster 2023-10-09T06:18:42.671673+0000 mgr.y (mgr.25723) 402 : cluster 0 pgmap v346: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:18:43.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:43.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:43.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:43.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:44.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:44.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:44.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:44.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 60949 60933 105.772 74.9375 0.0095867 0.00944873 2023-10-09T06:18:44.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:44.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:44.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:44.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:44.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:44.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:44.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:44.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:44.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:44.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:44.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:45.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:45.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:45.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 62049 62033 104.772 68.75 0.00676857 0.00953911 2023-10-09T06:18:45.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:45.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:45.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:45.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:45.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:45.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:45.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:45.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:45.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:45.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:45.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:45.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:45.737 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:45 smithi125 bash[77299]: cluster 2023-10-09T06:18:44.672280+0000 mgr.y (mgr.25723) 403 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:18:45.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:45.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:45 smithi155 bash[59495]: cluster 2023-10-09T06:18:44.672280+0000 mgr.y (mgr.25723) 403 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:18:45.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:45.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:45.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:45.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:46.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:46.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:46.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:46.158 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:45 smithi125 bash[80002]: cluster 2023-10-09T06:18:44.672280+0000 mgr.y (mgr.25723) 403 : cluster 0 pgmap v347: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:18:46.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 63047 63031 103.656 62.375 0.00994556 0.00963221 2023-10-09T06:18:46.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:46.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:46.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:46.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:46.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:46.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:46.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:46.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:46.744 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:46.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:46.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:46.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:46.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:47.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:47.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:47.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:47.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:47.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:47.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 64094 64078 102.676 65.4375 0.0158405 0.00973483 2023-10-09T06:18:47.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:47.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:47.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:47.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:47.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:47.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:47.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:47.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:47.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:47.801 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:47 smithi155 bash[59495]: cluster 2023-10-09T06:18:46.672988+0000 mgr.y (mgr.25723) 404 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:18:47.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:47.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:47.936 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:47 smithi125 bash[77299]: cluster 2023-10-09T06:18:46.672988+0000 mgr.y (mgr.25723) 404 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:18:47.936 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:47 smithi125 bash[80002]: cluster 2023-10-09T06:18:46.672988+0000 mgr.y (mgr.25723) 404 : cluster 0 pgmap v348: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:18:47.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:47.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:48.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:48.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:48.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:18:48.167466+0000 min lat: 0.00139442 max lat: 0.335425 avg lat: 0.00980081 2023-10-09T06:18:48.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:48.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 65288 65272 101.974 74.625 0.00937161 0.00980081 2023-10-09T06:18:48.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:48.331 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:48.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:18:48.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:48.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:48.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:48 smithi125 bash[77299]: audit 2023-10-09T06:18:48.564988+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:48.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:48.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:48.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:48 smithi155 bash[59495]: audit 2023-10-09T06:18:48.564988+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:48.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:48.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:48.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:49.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:49.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:49.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:48 smithi125 bash[80002]: audit 2023-10-09T06:18:48.564988+0000 mon.a (mon.0) 538 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:49.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 66411 66395 101.199 70.1875 0.0156673 0.0098764 2023-10-09T06:18:49.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:49.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:49.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:49.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:49.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:49.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:49.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:49.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:49.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:50.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:49 smithi125 bash[77299]: cluster 2023-10-09T06:18:48.674235+0000 mgr.y (mgr.25723) 405 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:18:50.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:49 smithi125 bash[80002]: cluster 2023-10-09T06:18:48.674235+0000 mgr.y (mgr.25723) 405 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:18:50.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:49 smithi155 bash[59495]: cluster 2023-10-09T06:18:48.674235+0000 mgr.y (mgr.25723) 405 : cluster 0 pgmap v349: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:18:50.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:50.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 67519 67503 100.438 69.25 0.00489596 0.00995069 2023-10-09T06:18:50.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:50.243 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:50.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:50.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:50.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:50.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:50.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:50.876 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:50.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:50.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:51.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:51.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:51.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:51.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:51.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 68721 68705 99.8492 75.125 0.0118075 0.0100094 2023-10-09T06:18:51.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:51.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:51.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:51.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:51.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:51.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:51.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:51.638 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:18:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:18:51] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:18:51.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:51.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:51.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:51.803 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:51 smithi155 bash[59495]: cluster 2023-10-09T06:18:50.674956+0000 mgr.y (mgr.25723) 406 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:18:51.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:51.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:51.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:51.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:52.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:52.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:52.124 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:51 smithi125 bash[77299]: cluster 2023-10-09T06:18:50.674956+0000 mgr.y (mgr.25723) 406 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:18:52.124 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:51 smithi125 bash[80002]: cluster 2023-10-09T06:18:50.674956+0000 mgr.y (mgr.25723) 406 : cluster 0 pgmap v350: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:18:52.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 69793 69777 99.1025 67 0.0102003 0.0100854 2023-10-09T06:18:52.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:52.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:52.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:52.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:52.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:52.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:52.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:52.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:52.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:52.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:52.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:52.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:52.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:52.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:52.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:53.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:53.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:53.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:53.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 70815 70799 98.3196 63.875 0.00484764 0.0101649 2023-10-09T06:18:53.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:53.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:53.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:53.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:53.520 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:18:53.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:53.557 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:18:53.612 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:18:53.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.644 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:18:53.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:53.682 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:18:53.726 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:18:53.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:53.759 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:53 smithi155 bash[59495]: cluster 2023-10-09T06:18:52.675597+0000 mgr.y (mgr.25723) 407 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T06:18:53.759 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:53 smithi155 bash[59495]: audit 2023-10-09T06:18:53.468445+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.125:0/4056256134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:53.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:18:53.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:53.797 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:18:53.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.802 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.803 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:53.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:53.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:53.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:53.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:54.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:53 smithi125 bash[77299]: cluster 2023-10-09T06:18:52.675597+0000 mgr.y (mgr.25723) 407 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T06:18:54.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:53 smithi125 bash[77299]: audit 2023-10-09T06:18:53.468445+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.125:0/4056256134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:54.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:53 smithi125 bash[80002]: cluster 2023-10-09T06:18:52.675597+0000 mgr.y (mgr.25723) 407 : cluster 0 pgmap v351: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T06:18:54.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:53 smithi125 bash[80002]: audit 2023-10-09T06:18:53.468445+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.125:0/4056256134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:54.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:54.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 71902 71886 97.659 67.9375 0.0210144 0.0102341 2023-10-09T06:18:54.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:54.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:54.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:54.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:54.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:54.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:54.783 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:18:54.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:54.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:54.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:54.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:54 smithi125 bash[77299]: audit 2023-10-09T06:18:54.649021+0000 mon.a (mon.0) 539 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:54 smithi125 bash[80002]: audit 2023-10-09T06:18:54.649021+0000 mon.a (mon.0) 539 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:55.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.057 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:54 smithi155 bash[59495]: audit 2023-10-09T06:18:54.649021+0000 mon.a (mon.0) 539 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:55.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:55.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 72718 72702 96.6662 51 0.00929581 0.0103397 2023-10-09T06:18:55.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:55.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:55.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:55.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:55.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:55.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:55.606 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:55.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:55.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:55.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:55.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:55.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:55.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.676214+0000 mgr.y (mgr.25723) 408 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T06:18:55.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.755096+0000 mon.a (mon.0) 540 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:55.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.755181+0000 mon.a (mon.0) 541 : cluster 1 Cluster is now healthy 2023-10-09T06:18:55.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: audit 2023-10-09T06:18:54.767104+0000 mon.a (mon.0) 542 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.767184+0000 mon.a (mon.0) 543 : cluster 0 osdmap e116: 8 total, 8 up, 7 in 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.778409+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.778476+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.780035+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.780144+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.786943+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[77299]: cluster 2023-10-09T06:18:54.856650+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.676214+0000 mgr.y (mgr.25723) 408 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T06:18:55.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.755096+0000 mon.a (mon.0) 540 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.755181+0000 mon.a (mon.0) 541 : cluster 1 Cluster is now healthy 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: audit 2023-10-09T06:18:54.767104+0000 mon.a (mon.0) 542 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.767184+0000 mon.a (mon.0) 543 : cluster 0 osdmap e116: 8 total, 8 up, 7 in 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.778409+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.778476+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.780035+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.780144+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.786943+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:55 smithi125 bash[80002]: cluster 2023-10-09T06:18:54.856650+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:55.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:55.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:55.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:56.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.676214+0000 mgr.y (mgr.25723) 408 : cluster 0 pgmap v352: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.755096+0000 mon.a (mon.0) 540 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.755181+0000 mon.a (mon.0) 541 : cluster 1 Cluster is now healthy 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: audit 2023-10-09T06:18:54.767104+0000 mon.a (mon.0) 542 : audit 1 from='client.? 172.21.15.125:0/3352548363' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.767184+0000 mon.a (mon.0) 543 : cluster 0 osdmap e116: 8 total, 8 up, 7 in 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.778409+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.778476+0000 osd.4 (osd.4) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.101 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.780035+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.780144+0000 osd.4 (osd.4) 38 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.786943+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:55 smithi155 bash[59495]: cluster 2023-10-09T06:18:54.856650+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e116 with expected crc 2023-10-09T06:18:56.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:56.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 73814 73798 96.0793 68.5 0.00871267 0.0104022 2023-10-09T06:18:56.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.256 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:56.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:56.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:56.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:56.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:56.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:56.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:56.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:56.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:56.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:57.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:57.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:57.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:57.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 74917 74901 95.5253 68.9375 0.037068 0.0104626 2023-10-09T06:18:57.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:57.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:57.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:57.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:57.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:57.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:57.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:57.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:57.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:57.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:57.770 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:57 smithi125 bash[77299]: cluster 2023-10-09T06:18:56.676717+0000 mgr.y (mgr.25723) 409 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:18:57.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:57.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:57.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:57.975 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:57 smithi155 bash[59495]: cluster 2023-10-09T06:18:56.676717+0000 mgr.y (mgr.25723) 409 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:18:57.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:57.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:58.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:58.158 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:57 smithi125 bash[80002]: cluster 2023-10-09T06:18:56.676717+0000 mgr.y (mgr.25723) 409 : cluster 0 pgmap v354: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:18:58.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 75997 75981 94.9646 67.5 0.00856224 0.0105247 2023-10-09T06:18:58.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:58.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:58.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:58.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:58.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:58.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:58.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:58.645 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:58 smithi125 bash[77299]: debug 2023-10-09T06:18:58.264+0000 7ff949fd9700 -1 mon.a@0(leader).osd e116 definitely_dead 0 2023-10-09T06:18:58.719 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:58.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:58.796 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.7. 2023-10-09T06:18:58.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:58.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:58 smithi155 bash[59495]: audit 2023-10-09T06:18:58.267226+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.125:0/3933101340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:58.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:58 smithi155 bash[59495]: audit 2023-10-09T06:18:58.269083+0000 mon.a (mon.0) 544 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:58.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:58.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:59.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:59.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:59.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:58 smithi125 bash[80002]: audit 2023-10-09T06:18:58.267226+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.125:0/3933101340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:59.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:58 smithi125 bash[80002]: audit 2023-10-09T06:18:58.269083+0000 mon.a (mon.0) 544 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:59.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:58 smithi125 bash[77299]: audit 2023-10-09T06:18:58.267226+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.125:0/3933101340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:59.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:58 smithi125 bash[77299]: audit 2023-10-09T06:18:58.269083+0000 mon.a (mon.0) 544 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:18:59.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:59.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 76694 76678 93.9567 43.5625 0.00493737 0.0105634 2023-10-09T06:18:59.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:59.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:59.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:59.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:59.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:59.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:59.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:18:59.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:59.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:59.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:59.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:18:59.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:59.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:00.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:00.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:00.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: cluster 2023-10-09T06:18:58.677222+0000 mgr.y (mgr.25723) 410 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:19:00.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: audit 2023-10-09T06:18:58.772364+0000 mon.a (mon.0) 545 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:00.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: cluster 2023-10-09T06:18:58.772528+0000 mon.a (mon.0) 546 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:19:00.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: cluster 2023-10-09T06:18:58.773944+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: cluster 2023-10-09T06:18:58.789119+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.779051+0000 mon.a (mon.0) 547 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:19:00.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: cluster 2023-10-09T06:18:58.677222+0000 mgr.y (mgr.25723) 410 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:19:00.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: audit 2023-10-09T06:18:58.772364+0000 mon.a (mon.0) 545 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:00.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: cluster 2023-10-09T06:18:58.772528+0000 mon.a (mon.0) 546 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:19:00.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: cluster 2023-10-09T06:18:58.773944+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: cluster 2023-10-09T06:18:58.789119+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:18:59 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.779051+0000 mon.a (mon.0) 547 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:19:00.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: cluster 2023-10-09T06:18:58.677222+0000 mgr.y (mgr.25723) 410 : cluster 0 pgmap v355: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:19:00.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: audit 2023-10-09T06:18:58.772364+0000 mon.a (mon.0) 545 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:00.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: cluster 2023-10-09T06:18:58.772528+0000 mon.a (mon.0) 546 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:19:00.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: cluster 2023-10-09T06:18:58.773944+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: cluster 2023-10-09T06:18:58.789119+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e117 with expected crc 2023-10-09T06:19:00.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:18:59 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.779051+0000 mon.a (mon.0) 547 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:19:00.156 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:00.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 76694 76678 92.1498 0 - 0.0105634 2023-10-09T06:19:00.193 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T06:19:00.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:00.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:00.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:00.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:00.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:00.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:00.617 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:00.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:00.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:00.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:00.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:00.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:00.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:00.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:00.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:01.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.781977+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.782105+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.782236+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.784594+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.785506+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.072 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.786242+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.786654+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.789434+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.789781+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.791910+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.793989+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:18:59.796099+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.073 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:00 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.779436+0000 mon.a (mon.0) 548 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:19:01.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:01.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.781977+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.782105+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.782236+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.784594+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.785506+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.786242+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.786654+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.789434+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.789781+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.791910+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.793989+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:18:59.796099+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.779436+0000 mon.a (mon.0) 548 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:19:01.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.781977+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.782105+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.782236+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.784594+0000 osd.4 (osd.4) 39 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.785506+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.786242+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.786654+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.789434+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.789781+0000 osd.4 (osd.4) 40 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.791910+0000 osd.4 (osd.4) 41 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.793989+0000 osd.4 (osd.4) 42 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:18:59.796099+0000 osd.4 (osd.4) 43 : cluster 3 failed to encode map e118 with expected crc 2023-10-09T06:19:01.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:00 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.779436+0000 mon.a (mon.0) 548 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:19:01.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 76739 76723 90.4643 1.40625 0.00439401 0.0110168 2023-10-09T06:19:01.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:01.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:01.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:01.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:01.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:01.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:01.481 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:01] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:19:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:01.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:01.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:01.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:01.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:01.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.677947+0000 mgr.y (mgr.25723) 411 : cluster 0 pgmap v358: 89 pgs: 18 peering, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 741 op/s 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.781037+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.781128+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.784068+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.784955+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.785064+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.788069+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:01 smithi155 bash[59495]: cluster 2023-10-09T06:19:00.788880+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:01.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:01.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:01.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.677947+0000 mgr.y (mgr.25723) 411 : cluster 0 pgmap v358: 89 pgs: 18 peering, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 741 op/s 2023-10-09T06:19:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.781037+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.781128+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.784068+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.784955+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.785064+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.788069+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[77299]: cluster 2023-10-09T06:19:00.788880+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.677947+0000 mgr.y (mgr.25723) 411 : cluster 0 pgmap v358: 89 pgs: 18 peering, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 741 op/s 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.781037+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.000 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.781128+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.784068+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.784955+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.785064+0000 osd.4 (osd.4) 44 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.788069+0000 osd.4 (osd.4) 45 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.001 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:01 smithi125 bash[80002]: cluster 2023-10-09T06:19:00.788880+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e119 with expected crc 2023-10-09T06:19:02.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:02.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:02.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:02.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:02.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 77423 77407 89.5806 42.75 0.0285956 0.0111577 2023-10-09T06:19:02.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:02.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:02.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:02.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:02.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:02.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:02.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:02.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:02.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:02.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.809481+0000 mon.a (mon.0) 549 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.813208+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.817113+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.818086+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.819197+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.820005+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:01.821079+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:02.822 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.810171+0000 mon.a (mon.0) 550 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:19:02.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:02.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:02.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:02.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:02.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:03.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:03.092 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.809481+0000 mon.a (mon.0) 549 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.813208+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.817113+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.818086+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.819197+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.820005+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:01.821079+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.093 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:02 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.810171+0000 mon.a (mon.0) 550 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:19:03.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:03.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:03.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.809481+0000 mon.a (mon.0) 549 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:19:03.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.813208+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.817113+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.818086+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.819197+0000 osd.4 (osd.4) 46 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.820005+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:01.821079+0000 osd.4 (osd.4) 47 : cluster 3 failed to encode map e120 with expected crc 2023-10-09T06:19:03.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:02 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.810171+0000 mon.a (mon.0) 550 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:19:03.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 78251 78235 88.8927 51.75 0.00698084 0.0112435 2023-10-09T06:19:03.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:03.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:03.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:03.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:03.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:03.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:03.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:03.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:03.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:03.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:03.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:03.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.678450+0000 mgr.y (mgr.25723) 412 : cluster 0 pgmap v361: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 18 peering, 67 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 753 op/s; 9459/229314 objects degraded (4.125%); 4852/229314 objects misplaced (2.116%) 2023-10-09T06:19:03.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.814606+0000 mon.a (mon.0) 551 : cluster 3 Health check failed: Degraded data redundancy: 9459/229314 objects degraded (4.125%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:03.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.814837+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.817492+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.817899+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.819723+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.820416+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.821484+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.822278+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: cluster 2023-10-09T06:19:02.832745+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: audit 2023-10-09T06:19:03.571719+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:03.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[77299]: audit 2023-10-09T06:19:03.573077+0000 mon.a (mon.0) 553 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:03.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:03.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.678450+0000 mgr.y (mgr.25723) 412 : cluster 0 pgmap v361: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 18 peering, 67 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 753 op/s; 9459/229314 objects degraded (4.125%); 4852/229314 objects misplaced (2.116%) 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.814606+0000 mon.a (mon.0) 551 : cluster 3 Health check failed: Degraded data redundancy: 9459/229314 objects degraded (4.125%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.814837+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.817492+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.817899+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.819723+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.820416+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.821484+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.822278+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: cluster 2023-10-09T06:19:02.832745+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: audit 2023-10-09T06:19:03.571719+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:03 smithi155 bash[59495]: audit 2023-10-09T06:19:03.573077+0000 mon.a (mon.0) 553 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:03.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:03.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:04.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:04.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.678450+0000 mgr.y (mgr.25723) 412 : cluster 0 pgmap v361: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 18 peering, 67 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 753 op/s; 9459/229314 objects degraded (4.125%); 4852/229314 objects misplaced (2.116%) 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.814606+0000 mon.a (mon.0) 551 : cluster 3 Health check failed: Degraded data redundancy: 9459/229314 objects degraded (4.125%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.814837+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.817492+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.817899+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.819723+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.820416+0000 osd.4 (osd.4) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.821484+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.822278+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: cluster 2023-10-09T06:19:02.832745+0000 osd.4 (osd.4) 49 : cluster 3 failed to encode map e121 with expected crc 2023-10-09T06:19:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: audit 2023-10-09T06:19:03.571719+0000 mon.a (mon.0) 552 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:03 smithi125 bash[80002]: audit 2023-10-09T06:19:03.573077+0000 mon.a (mon.0) 553 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:04.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:04.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 79055 79039 88.2026 50.25 0.0126641 0.01133 2023-10-09T06:19:04.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:04.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:04.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:04.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:04.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:04.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:04.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:04.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:04.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:04.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:04.834 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:19:04.834 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:19:04.834 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:19:04.834 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:19:04.834 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:19:04.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:04.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:04.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:04.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:04.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:05.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:05.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:05.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 79885 79869 87.5652 51.875 0.0114948 0.0114136 2023-10-09T06:19:05.194 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:19:05.194 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:19:05.195 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-09T06:19:05.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:19:05.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:05.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:05.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:05.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:05.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:05.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:05.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:05.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:05.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:05.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:05 smithi125 bash[77299]: cluster 2023-10-09T06:19:04.679178+0000 mgr.y (mgr.25723) 413 : cluster 0 pgmap v363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 711 KiB/s rd, 39 MiB/s wr, 865 op/s; 9469/233298 objects degraded (4.059%); 14407/233298 objects misplaced (6.175%); 888 KiB/s, 2 keys/s, 22 objects/s recovering 2023-10-09T06:19:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:05.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:05.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:05.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:06.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:19:06.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:06.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:05 smithi155 bash[59495]: cluster 2023-10-09T06:19:04.679178+0000 mgr.y (mgr.25723) 413 : cluster 0 pgmap v363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 711 KiB/s rd, 39 MiB/s wr, 865 op/s; 9469/233298 objects degraded (4.059%); 14407/233298 objects misplaced (6.175%); 888 KiB/s, 2 keys/s, 22 objects/s recovering 2023-10-09T06:19:06.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:06.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:05 smithi125 bash[80002]: cluster 2023-10-09T06:19:04.679178+0000 mgr.y (mgr.25723) 413 : cluster 0 pgmap v363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 711 KiB/s rd, 39 MiB/s wr, 865 op/s; 9469/233298 objects degraded (4.059%); 14407/233298 objects misplaced (6.175%); 888 KiB/s, 2 keys/s, 22 objects/s recovering 2023-10-09T06:19:06.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 80688 80672 86.9207 50.1875 0.0243543 0.0114983 2023-10-09T06:19:06.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:06.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:06.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:06.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:06.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:06.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:06.546 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:06.547 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:06.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:06.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:06.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:06.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:06.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:06.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:06.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:07.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:07.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:07.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:07.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 81502 81486 86.3097 50.875 0.0127586 0.0115795 2023-10-09T06:19:07.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:07.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:07.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:07.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:07.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:07.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:07.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:07.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:07.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:07.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:07 smithi125 bash[77299]: cluster 2023-10-09T06:19:06.679865+0000 mgr.y (mgr.25723) 414 : cluster 0 pgmap v364: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 581 KiB/s rd, 52 MiB/s wr, 1.02k op/s; 9469/239010 objects degraded (3.962%); 14407/239010 objects misplaced (6.028%); 726 KiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T06:19:07.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:07.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:07.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:07.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:07.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:08.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:08.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:07 smithi125 bash[80002]: cluster 2023-10-09T06:19:06.679865+0000 mgr.y (mgr.25723) 414 : cluster 0 pgmap v364: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 581 KiB/s rd, 52 MiB/s wr, 1.02k op/s; 9469/239010 objects degraded (3.962%); 14407/239010 objects misplaced (6.028%); 726 KiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T06:19:08.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:08.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:07 smithi155 bash[59495]: cluster 2023-10-09T06:19:06.679865+0000 mgr.y (mgr.25723) 414 : cluster 0 pgmap v364: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 581 KiB/s rd, 52 MiB/s wr, 1.02k op/s; 9469/239010 objects degraded (3.962%); 14407/239010 objects misplaced (6.028%); 726 KiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T06:19:08.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:19:08.169362+0000 min lat: 0.00139442 max lat: 2.36825 avg lat: 0.0116557 2023-10-09T06:19:08.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:19:08.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 82326 82310 85.7295 51.5 0.00777956 0.0116557 2023-10-09T06:19:08.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.233 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:08.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:08.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:08.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:08.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:08.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:08.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:08.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:08 smithi155 bash[59495]: cluster 2023-10-09T06:19:08.611991+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 9469/239010 objects degraded (3.962%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:08.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:08 smithi125 bash[80002]: cluster 2023-10-09T06:19:08.611991+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 9469/239010 objects degraded (3.962%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:08 smithi125 bash[77299]: cluster 2023-10-09T06:19:08.611991+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 9469/239010 objects degraded (3.962%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:08.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:08.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:09.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:09.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:09.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 83125 83109 85.1427 49.9375 0.0138087 0.0117384 2023-10-09T06:19:09.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:09.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:09.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:09.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:09.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:09.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:09.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:09.598 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:09.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:09.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:09.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:09.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:09.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:09 smithi125 bash[77299]: cluster 2023-10-09T06:19:08.680444+0000 mgr.y (mgr.25723) 415 : cluster 0 pgmap v365: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 441 KiB/s rd, 44 MiB/s wr, 849 op/s; 9469/241347 objects degraded (3.923%); 14407/241347 objects misplaced (5.969%); 551 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T06:19:09.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:09 smithi125 bash[80002]: cluster 2023-10-09T06:19:08.680444+0000 mgr.y (mgr.25723) 415 : cluster 0 pgmap v365: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 441 KiB/s rd, 44 MiB/s wr, 849 op/s; 9469/241347 objects degraded (3.923%); 14407/241347 objects misplaced (5.969%); 551 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T06:19:09.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:09.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:09.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:10.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:10.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:10.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:10.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:09 smithi155 bash[59495]: cluster 2023-10-09T06:19:08.680444+0000 mgr.y (mgr.25723) 415 : cluster 0 pgmap v365: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 441 KiB/s rd, 44 MiB/s wr, 849 op/s; 9469/241347 objects degraded (3.923%); 14407/241347 objects misplaced (5.969%); 551 KiB/s, 1 keys/s, 14 objects/s recovering 2023-10-09T06:19:10.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 83879 83863 84.5294 47.125 0.0389892 0.0118235 2023-10-09T06:19:10.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:10.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:10.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:10.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:10.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:10.624 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:10.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:10.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:10.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:10.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:10.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:10.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:10.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:10.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:11.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:11.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 84680 84664 83.9823 50.0625 0.010145 0.0119014 2023-10-09T06:19:11.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:11.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:11.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:11.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:11.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:11.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:11.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:11.692 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:11] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-10-09T06:19:11.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:11.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:11.845 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:11 smithi155 bash[59495]: cluster 2023-10-09T06:19:10.681307+0000 mgr.y (mgr.25723) 416 : cluster 0 pgmap v366: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 393 KiB/s rd, 38 MiB/s wr, 733 op/s; 9469/245274 objects degraded (3.861%); 14331/245274 objects misplaced (5.843%); 1.0 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T06:19:11.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:11.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:11.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:12.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:12.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:12.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:11 smithi125 bash[77299]: cluster 2023-10-09T06:19:10.681307+0000 mgr.y (mgr.25723) 416 : cluster 0 pgmap v366: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 393 KiB/s rd, 38 MiB/s wr, 733 op/s; 9469/245274 objects degraded (3.861%); 14331/245274 objects misplaced (5.843%); 1.0 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T06:19:12.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:11 smithi125 bash[80002]: cluster 2023-10-09T06:19:10.681307+0000 mgr.y (mgr.25723) 416 : cluster 0 pgmap v366: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 393 KiB/s rd, 38 MiB/s wr, 733 op/s; 9469/245274 objects degraded (3.861%); 14331/245274 objects misplaced (5.843%); 1.0 MiB/s, 1 keys/s, 21 objects/s recovering 2023-10-09T06:19:12.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 85427 85411 83.3995 46.6875 0.00574471 0.0119834 2023-10-09T06:19:12.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:12.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:12.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:12.580 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:12.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:12.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:12.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:13.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:13.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:13.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 86005 85989 82.6722 36.125 0.0187573 0.0120886 2023-10-09T06:19:13.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:13.244 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:13.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:13.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:13.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:13.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:13.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:13.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:13 smithi155 bash[59495]: cluster 2023-10-09T06:19:12.682083+0000 mgr.y (mgr.25723) 417 : cluster 0 pgmap v367: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 348 KiB/s rd, 46 MiB/s wr, 850 op/s; 9469/251247 objects degraded (3.769%); 14331/251247 objects misplaced (5.704%); 922 KiB/s, 0 keys/s, 18 objects/s recovering 2023-10-09T06:19:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:13 smithi155 bash[59495]: cluster 2023-10-09T06:19:13.613071+0000 mon.a (mon.0) 555 : cluster 3 Health check update: Degraded data redundancy: 9469/251247 objects degraded (3.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:13 smithi155 bash[59495]: audit 2023-10-09T06:19:13.619333+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:13 smithi155 bash[59495]: audit 2023-10-09T06:19:13.797513+0000 mon.a (mon.0) 557 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:13.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:13.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:14.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:14.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[77299]: cluster 2023-10-09T06:19:12.682083+0000 mgr.y (mgr.25723) 417 : cluster 0 pgmap v367: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 348 KiB/s rd, 46 MiB/s wr, 850 op/s; 9469/251247 objects degraded (3.769%); 14331/251247 objects misplaced (5.704%); 922 KiB/s, 0 keys/s, 18 objects/s recovering 2023-10-09T06:19:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[77299]: cluster 2023-10-09T06:19:13.613071+0000 mon.a (mon.0) 555 : cluster 3 Health check update: Degraded data redundancy: 9469/251247 objects degraded (3.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[77299]: audit 2023-10-09T06:19:13.619333+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:14.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[77299]: audit 2023-10-09T06:19:13.797513+0000 mon.a (mon.0) 557 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[80002]: cluster 2023-10-09T06:19:12.682083+0000 mgr.y (mgr.25723) 417 : cluster 0 pgmap v367: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 348 KiB/s rd, 46 MiB/s wr, 850 op/s; 9469/251247 objects degraded (3.769%); 14331/251247 objects misplaced (5.704%); 922 KiB/s, 0 keys/s, 18 objects/s recovering 2023-10-09T06:19:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[80002]: cluster 2023-10-09T06:19:13.613071+0000 mon.a (mon.0) 555 : cluster 3 Health check update: Degraded data redundancy: 9469/251247 objects degraded (3.769%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[80002]: audit 2023-10-09T06:19:13.619333+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:13 smithi125 bash[80002]: audit 2023-10-09T06:19:13.797513+0000 mon.a (mon.0) 557 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:14.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 86751 86735 82.126 46.625 0.00698874 0.0121696 2023-10-09T06:19:14.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:14.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:14.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:14.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:14.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:14.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:14.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:14.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:14.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:14.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:14.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:14.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:14.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:14.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:14.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:15.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:15.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:15.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 87376 87360 81.4832 39.0625 0.00552866 0.0122665 2023-10-09T06:19:15.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:15.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:15.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:15.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:15.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:15.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:15.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:15.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:15.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:15.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:15.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:15.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:15.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:15.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:15.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:15.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:16.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:16.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:15 smithi125 bash[77299]: cluster 2023-10-09T06:19:14.682835+0000 mgr.y (mgr.25723) 418 : cluster 0 pgmap v368: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 293 KiB/s rd, 48 MiB/s wr, 868 op/s; 9469/256692 objects degraded (3.689%); 14247/256692 objects misplaced (5.550%); 1.2 MiB/s, 0 keys/s, 22 objects/s recovering 2023-10-09T06:19:16.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:15 smithi125 bash[80002]: cluster 2023-10-09T06:19:14.682835+0000 mgr.y (mgr.25723) 418 : cluster 0 pgmap v368: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 293 KiB/s rd, 48 MiB/s wr, 868 op/s; 9469/256692 objects degraded (3.689%); 14247/256692 objects misplaced (5.550%); 1.2 MiB/s, 0 keys/s, 22 objects/s recovering 2023-10-09T06:19:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:15 smithi155 bash[59495]: cluster 2023-10-09T06:19:14.682835+0000 mgr.y (mgr.25723) 418 : cluster 0 pgmap v368: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 293 KiB/s rd, 48 MiB/s wr, 868 op/s; 9469/256692 objects degraded (3.689%); 14247/256692 objects misplaced (5.550%); 1.2 MiB/s, 0 keys/s, 22 objects/s recovering 2023-10-09T06:19:16.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:16.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 88081 88065 80.9328 44.0625 0.0108624 0.0123491 2023-10-09T06:19:16.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:16.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:16.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:16.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:16.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:16.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:16.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:16.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:16.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:16.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:16.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:16.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:16.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:17.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:17.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:17.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 88853 88837 80.4591 48.25 0.00633942 0.0124229 2023-10-09T06:19:17.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:17.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:17.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:17.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:17.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:17.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:17.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:17.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:17.867 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:17 smithi155 bash[59495]: cluster 2023-10-09T06:19:16.683412+0000 mgr.y (mgr.25723) 419 : cluster 0 pgmap v369: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 9469/261951 objects degraded (3.615%); 14247/261951 objects misplaced (5.439%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:17.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:17.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:17.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:18.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:18.025 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:17 smithi125 bash[77299]: cluster 2023-10-09T06:19:16.683412+0000 mgr.y (mgr.25723) 419 : cluster 0 pgmap v369: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 9469/261951 objects degraded (3.615%); 14247/261951 objects misplaced (5.439%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:18.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:17 smithi125 bash[80002]: cluster 2023-10-09T06:19:16.683412+0000 mgr.y (mgr.25723) 419 : cluster 0 pgmap v369: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 9469/261951 objects degraded (3.615%); 14247/261951 objects misplaced (5.439%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:18.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:18.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:18.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 89626 89610 79.9992 48.3125 0.0145463 0.0124924 2023-10-09T06:19:18.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:18.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:18.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:18.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:18.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:18.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:18.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:18.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:18.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:18.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:18.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:18.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:18.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:18.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:19.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:19.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:19.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:19.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:18 smithi155 bash[59495]: audit 2023-10-09T06:19:18.565794+0000 mon.a (mon.0) 558 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:19.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:18 smithi155 bash[59495]: cluster 2023-10-09T06:19:18.614319+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 9469/261951 objects degraded (3.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:19.145 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:19.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 90315 90299 79.4789 43.0625 0.0173536 0.0125747 2023-10-09T06:19:19.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:19.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:18 smithi125 bash[77299]: audit 2023-10-09T06:19:18.565794+0000 mon.a (mon.0) 558 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:19.261 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:18 smithi125 bash[77299]: cluster 2023-10-09T06:19:18.614319+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 9469/261951 objects degraded (3.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:19.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:18 smithi125 bash[80002]: audit 2023-10-09T06:19:18.565794+0000 mon.a (mon.0) 558 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:19.262 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:18 smithi125 bash[80002]: cluster 2023-10-09T06:19:18.614319+0000 mon.a (mon.0) 559 : cluster 3 Health check update: Degraded data redundancy: 9469/261951 objects degraded (3.615%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:19.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:19.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:19.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:19.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:19.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:19.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:19.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:19.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:19.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:19.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:19.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:20.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:20.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:20.123 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:19 smithi125 bash[80002]: cluster 2023-10-09T06:19:18.683948+0000 mgr.y (mgr.25723) 420 : cluster 0 pgmap v370: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 9469/263958 objects degraded (3.587%); 14247/263958 objects misplaced (5.397%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:20.124 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:19 smithi125 bash[77299]: cluster 2023-10-09T06:19:18.683948+0000 mgr.y (mgr.25723) 420 : cluster 0 pgmap v370: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 9469/263958 objects degraded (3.587%); 14247/263958 objects misplaced (5.397%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:20.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:20.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:19 smithi155 bash[59495]: cluster 2023-10-09T06:19:18.683948+0000 mgr.y (mgr.25723) 420 : cluster 0 pgmap v370: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s; 9469/263958 objects degraded (3.587%); 14247/263958 objects misplaced (5.397%); 853 KiB/s, 13 objects/s recovering 2023-10-09T06:19:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:20.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 90895 90879 78.8784 36.25 0.0435512 0.0126708 2023-10-09T06:19:20.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:20.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:20.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:20.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:20.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:20.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:20.500 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:19:20.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:20.545 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:19:20.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:20.579 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:19:20.614 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:19:20.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:20.653 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:19:20.676 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:20.680 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:19:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:20.713 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:19:20.738 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:19:20.739 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.740 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:20.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:20.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:20.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:20.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:20.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 91515 91499 78.3287 38.75 0.0171441 0.0127598 2023-10-09T06:19:21.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.354 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:21] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-10-09T06:19:21.376 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.401 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:19:21.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:21.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: audit 2023-10-09T06:19:20.370817+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: audit 2023-10-09T06:19:20.382488+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: audit 2023-10-09T06:19:20.448995+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.125:0/1003039970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: cluster 2023-10-09T06:19:20.684611+0000 mgr.y (mgr.25723) 421 : cluster 0 pgmap v371: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 9469/267108 objects degraded (3.545%); 14169/267108 objects misplaced (5.305%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: audit 2023-10-09T06:19:21.338803+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.125:0/620610547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[77299]: audit 2023-10-09T06:19:21.339426+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: audit 2023-10-09T06:19:20.370817+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: audit 2023-10-09T06:19:20.382488+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: audit 2023-10-09T06:19:20.448995+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.125:0/1003039970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: cluster 2023-10-09T06:19:20.684611+0000 mgr.y (mgr.25723) 421 : cluster 0 pgmap v371: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 9469/267108 objects degraded (3.545%); 14169/267108 objects misplaced (5.305%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: audit 2023-10-09T06:19:21.338803+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.125:0/620610547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:21 smithi125 bash[80002]: audit 2023-10-09T06:19:21.339426+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: audit 2023-10-09T06:19:20.370817+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: audit 2023-10-09T06:19:20.382488+0000 mon.a (mon.0) 561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: audit 2023-10-09T06:19:20.448995+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.125:0/1003039970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: cluster 2023-10-09T06:19:20.684611+0000 mgr.y (mgr.25723) 421 : cluster 0 pgmap v371: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 9469/267108 objects degraded (3.545%); 14169/267108 objects misplaced (5.305%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: audit 2023-10-09T06:19:21.338803+0000 mon.c (mon.2) 43 : audit 1 from='client.? 172.21.15.125:0/620610547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:21 smithi155 bash[59495]: audit 2023-10-09T06:19:21.339426+0000 mon.a (mon.0) 562 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:21.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:21.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:21.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:21.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:21.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:21.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:21.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:22.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:22.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:22.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 92243 92227 77.885 45.5 0.00483155 0.0128298 2023-10-09T06:19:22.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:22.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:22.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:22.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:22.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:22.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:22.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.379900+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:22.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: audit 2023-10-09T06:19:21.390230+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:22.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.390298+0000 mon.a (mon.0) 565 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:19:22.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.392256+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.395696+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.395887+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[77299]: cluster 2023-10-09T06:19:21.406590+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:22.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.379900+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: audit 2023-10-09T06:19:21.390230+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.390298+0000 mon.a (mon.0) 565 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:19:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.392256+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.395696+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.395887+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:22 smithi125 bash[80002]: cluster 2023-10-09T06:19:21.406590+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.379900+0000 mon.a (mon.0) 563 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: audit 2023-10-09T06:19:21.390230+0000 mon.a (mon.0) 564 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.390298+0000 mon.a (mon.0) 565 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.392256+0000 osd.4 (osd.4) 50 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.395696+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.395887+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:22 smithi155 bash[59495]: cluster 2023-10-09T06:19:21.406590+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e122 with expected crc 2023-10-09T06:19:22.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:22.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:22.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:22.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:23.002 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:23.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:23.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:23.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 92991 92975 77.4698 46.75 0.0135571 0.0129017 2023-10-09T06:19:23.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:23.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:23 smithi155 bash[59495]: cluster 2023-10-09T06:19:22.685230+0000 mgr.y (mgr.25723) 422 : cluster 0 pgmap v373: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 9469/272373 objects degraded (3.476%); 14169/272373 objects misplaced (5.202%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:23.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:23.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:23.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:23.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:23 smithi125 bash[77299]: cluster 2023-10-09T06:19:22.685230+0000 mgr.y (mgr.25723) 422 : cluster 0 pgmap v373: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 9469/272373 objects degraded (3.476%); 14169/272373 objects misplaced (5.202%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:23.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:23 smithi125 bash[80002]: cluster 2023-10-09T06:19:22.685230+0000 mgr.y (mgr.25723) 422 : cluster 0 pgmap v373: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 9469/272373 objects degraded (3.476%); 14169/272373 objects misplaced (5.202%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:23.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:23.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:23.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:23.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:23.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:23.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:23.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:24.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:24.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:24.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 93710 93694 77.0417 44.9375 0.0168594 0.0129735 2023-10-09T06:19:24.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:24.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:24.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:24.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:24.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:24 smithi125 bash[77299]: cluster 2023-10-09T06:19:23.615438+0000 mon.a (mon.0) 566 : cluster 3 Health check update: Degraded data redundancy: 9469/272373 objects degraded (3.476%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:24.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:24.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:24.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:24.604 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:24 smithi155 bash[59495]: cluster 2023-10-09T06:19:23.615438+0000 mon.a (mon.0) 566 : cluster 3 Health check update: Degraded data redundancy: 9469/272373 objects degraded (3.476%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:24.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:24.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:24 smithi125 bash[80002]: cluster 2023-10-09T06:19:23.615438+0000 mon.a (mon.0) 566 : cluster 3 Health check update: Degraded data redundancy: 9469/272373 objects degraded (3.476%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:24.715 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:19:24.722 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:19:24.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:24.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:24.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:24.882 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:24.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:24.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:25.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:25.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 94420 94404 76.6174 44.375 0.0156915 0.013046 2023-10-09T06:19:25.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:25.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:25.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:25.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:25.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:25.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:25.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:25 smithi125 bash[77299]: cluster 2023-10-09T06:19:24.685936+0000 mgr.y (mgr.25723) 423 : cluster 0 pgmap v374: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 9469/277419 objects degraded (3.413%); 14091/277419 objects misplaced (5.079%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:25.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:25 smithi125 bash[77299]: audit 2023-10-09T06:19:24.686647+0000 mgr.y (mgr.25723) 424 : audit 0 from='client.44729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:25.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:25 smithi125 bash[80002]: cluster 2023-10-09T06:19:24.685936+0000 mgr.y (mgr.25723) 423 : cluster 0 pgmap v374: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 9469/277419 objects degraded (3.413%); 14091/277419 objects misplaced (5.079%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:25.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:25 smithi125 bash[80002]: audit 2023-10-09T06:19:24.686647+0000 mgr.y (mgr.25723) 424 : audit 0 from='client.44729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:25.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:25 smithi155 bash[59495]: cluster 2023-10-09T06:19:24.685936+0000 mgr.y (mgr.25723) 423 : cluster 0 pgmap v374: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 9469/277419 objects degraded (3.413%); 14091/277419 objects misplaced (5.079%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:25.686 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:25 smithi155 bash[59495]: audit 2023-10-09T06:19:24.686647+0000 mgr.y (mgr.25723) 424 : audit 0 from='client.44729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:25.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:25.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:25.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:25.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:25.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:26.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:26.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:26.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 95202 95186 76.2617 48.875 0.00898111 0.0131067 2023-10-09T06:19:26.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:26.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:26.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:26.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:26.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:26.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:26.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:26.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:26.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:26.776 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":374,"stamp":"2023-10-09T06:19:24.685420+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6063265715,"num_objects":92473,"num_object_clones":0,"num_object_copies":277419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9469,"num_objects_misplaced":14091,"num_objects_unfound":0,"num_objects_dirty":92473,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":93551,"num_write_kb":5931639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":469,"num_bytes_recovered":25766420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93615,"ondisk_log_size":93615,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19504532,"kb_used_data":17969364,"kb_used_omap":30,"kb_used_meta":1535073,"kb_avail":730456684,"statfs":{"total":767960285184,"available":747987644416,"internally_reserved":0,"allocated":18400628736,"data_stored":18396310752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1571914863},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,3,5,0,4,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":452788224,"num_objects":6909,"num_object_clones":0,"num_object_copies":20727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-156,"num_objects_unfound":0,"num_objects_dirty":6909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6909,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":10223616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003118"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":103,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.851222+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:19:02.851222+0000","last_peered":"2023-10-09T06:19:02.851222+0000","last_clean":"2023-10-09T06:19:02.851222+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:19:02.851222+0000","last_undegraded":"2023-10-09T06:19:02.851222+0000","last_fullsized":"2023-10-09T06:19:02.851222+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.812343+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:19:01.812343+0000","last_peered":"2023-10-09T06:19:01.812343+0000","last_clean":"2023-10-09T06:19:01.812343+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:19:01.812343+0000","last_undegraded":"2023-10-09T06:19:01.812343+0000","last_fullsized":"2023-10-09T06:19:01.812343+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.812832+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:19:02.812832+0000","last_peered":"2023-10-09T06:19:02.812832+0000","last_clean":"2023-10-09T06:19:02.812832+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:19:02.812832+0000","last_undegraded":"2023-10-09T06:19:02.812832+0000","last_fullsized":"2023-10-09T06:19:02.812832+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:12.828093+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:19:12.828093+0000","last_peered":"2023-10-09T06:19:12.828093+0000","last_clean":"2023-10-09T06:19:12.828093+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:19:12.828093+0000","last_undegraded":"2023-10-09T06:19:12.828093+0000","last_fullsized":"2023-10-09T06:19:12.828093+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.811205+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:19:01.811205+0000","last_peered":"2023-10-09T06:19:01.811205+0000","last_clean":"2023-10-09T06:19:01.811205+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:19:01.811205+0000","last_undegraded":"2023-10-09T06:19:01.811205+0000","last_fullsized":"2023-10-09T06:19:01.811205+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.835133+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:19:02.835133+0000","last_peered":"2023-10-09T06:19:02.835133+0000","last_clean":"2023-10-09T06:19:02.835133+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:19:02.835133+0000","last_undegraded":"2023-10-09T06:19:02.835133+0000","last_fullsized":"2023-10-09T06:19:02.835133+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.822355+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:19:02.822355+0000","last_peered":"2023-10-09T06:19:02.822355+0000","last_clean":"2023-10-09T06:19:02.822355+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:19:02.822355+0000","last_undegraded":"2023-10-09T06:19:02.822355+0000","last_fullsized":"2023-10-09T06:19:02.822355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.827299+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:19:02.827299+0000","last_peered":"2023-10-09T06:19:02.827299+0000","last_clean":"2023-10-09T06:19:02.827299+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:19:02.827299+0000","last_undegraded":"2023-10-09T06:19:02.827299+0000","last_fullsized":"2023-10-09T06:19:02.827299+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.836745+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:19:02.836745+0000","last_peered":"2023-10-09T06:19:02.836745+0000","last_clean":"2023-10-09T06:19:02.836745+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:19:02.836745+0000","last_undegraded":"2023-10-09T06:19:02.836745+0000","last_fullsized":"2023-10-09T06:19:02.836745+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":99,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.828022+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:19:02.828022+0000","last_peered":"2023-10-09T06:19:02.828022+0000","last_clean":"2023-10-09T06:19:02.828022+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:19:02.828022+0000","last_undegraded":"2023-10-09T06:19:02.828022+0000","last_fullsized":"2023-10-09T06:19:02.828022+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":20,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.834110+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:19:02.834110+0000","last_peered":"2023-10-09T06:19:02.834110+0000","last_clean":"2023-10-09T06:19:02.834110+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:19:02.834110+0000","last_undegraded":"2023-10-09T06:19:02.834110+0000","last_fullsized":"2023-10-09T06:19:02.834110+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810359+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:19:02.810359+0000","last_peered":"2023-10-09T06:19:02.810359+0000","last_clean":"2023-10-09T06:19:02.810359+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:19:02.810359+0000","last_undegraded":"2023-10-09T06:19:02.810359+0000","last_fullsized":"2023-10-09T06:19:02.810359+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":132,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.846268+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:19:02.846268+0000","last_peered":"2023-10-09T06:19:02.846268+0000","last_clean":"2023-10-09T06:19:02.846268+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:19:02.846268+0000","last_undegraded":"2023-10-09T06:19:02.846268+0000","last_fullsized":"2023-10-09T06:19:02.846268+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":86,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.845787+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:19:02.845787+0000","last_peered":"2023-10-09T06:19:02.845787+0000","last_clean":"2023-10-09T06:19:02.845787+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:19:02.845787+0000","last_undegraded":"2023-10-09T06:19:02.845787+0000","last_fullsized":"2023-10-09T06:19:02.845787+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":20,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.829292+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:19:02.829292+0000","last_peered":"2023-10-09T06:19:02.829292+0000","last_clean":"2023-10-09T06:19:02.829292+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:19:02.829292+0000","last_undegraded":"2023-10-09T06:19:02.829292+0000","last_fullsized":"2023-10-09T06:19:02.829292+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.827584+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:19:02.827584+0000","last_peered":"2023-10-09T06:19:02.827584+0000","last_clean":"2023-10-09T06:19:02.827584+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:19:02.827584+0000","last_undegraded":"2023-10-09T06:19:02.827584+0000","last_fullsized":"2023-10-09T06:19:02.827584+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.822256+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:19:02.822256+0000","last_peered":"2023-10-09T06:19:02.822256+0000","last_clean":"2023-10-09T06:19:02.822256+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:19:02.822256+0000","last_undegraded":"2023-10-09T06:19:02.822256+0000","last_fullsized":"2023-10-09T06:19:02.822256+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":104,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.836038+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:19:02.836038+0000","last_peered":"2023-10-09T06:19:02.836038+0000","last_clean":"2023-10-09T06:19:02.836038+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:19:02.836038+0000","last_undegraded":"2023-10-09T06:19:02.836038+0000","last_fullsized":"2023-10-09T06:19:02.836038+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.822246+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:19:02.822246+0000","last_peered":"2023-10-09T06:19:02.822246+0000","last_clean":"2023-10-09T06:19:02.822246+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:19:02.822246+0000","last_undegraded":"2023-10-09T06:19:02.822246+0000","last_fullsized":"2023-10-09T06:19:02.822246+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.821678+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:19:02.821678+0000","last_peered":"2023-10-09T06:19:02.821678+0000","last_clean":"2023-10-09T06:19:02.821678+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:19:02.821678+0000","last_undegraded":"2023-10-09T06:19:02.821678+0000","last_fullsized":"2023-10-09T06:19:02.821678+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":96,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.834290+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:19:02.834290+0000","last_peered":"2023-10-09T06:19:02.834290+0000","last_clean":"2023-10-09T06:19:02.834290+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:19:02.834290+0000","last_undegraded":"2023-10-09T06:19:02.834290+0000","last_fullsized":"2023-10-09T06:19:02.834290+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":121,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.812571+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:19:01.812571+0000","last_peered":"2023-10-09T06:19:01.812571+0000","last_clean":"2023-10-09T06:19:01.812571+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:19:01.812571+0000","last_undegraded":"2023-10-09T06:19:01.812571+0000","last_fullsized":"2023-10-09T06:19:01.812571+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"122'5783","reported_seq":5812,"reported_epoch":122,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T06:19:21.445443+0000","last_change":"2023-10-09T06:19:01.092933+0000","last_active":"2023-10-09T06:19:21.445443+0000","last_peered":"2023-10-09T06:19:21.445443+0000","last_clean":"2023-10-09T06:18:58.745871+0000","last_became_active":"2023-10-09T06:19:01.079126+0000","last_became_peered":"2023-10-09T06:19:01.079126+0000","last_unstale":"2023-10-09T06:19:21.445443+0000","last_undegraded":"2023-10-09T06:19:21.445443+0000","last_fullsized":"2023-10-09T06:19:21.445443+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:15.303333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378994688,"num_objects":5783,"num_object_clones":0,"num_object_copies":17349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4852,"num_objects_unfound":0,"num_objects_dirty":5783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5783,"num_write_kb":370112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":931}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":18,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.829180+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:19:02.829180+0000","last_peered":"2023-10-09T06:19:02.829180+0000","last_clean":"2023-10-09T06:19:02.829180+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:19:02.829180+0000","last_undegraded":"2023-10-09T06:19:02.829180+0000","last_fullsized":"2023-10-09T06:19:02.829180+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.834811+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:19:02.834811+0000","last_peered":"2023-10-09T06:19:02.834811+0000","last_clean":"2023-10-09T06:19:02.834811+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:19:02.834811+0000","last_undegraded":"2023-10-09T06:19:02.834811+0000","last_fullsized":"2023-10-09T06:19:02.834811+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"121'5582","reported_seq":5607,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:20.976111+0000","last_change":"2023-10-09T06:17:45.111620+0000","last_active":"2023-10-09T06:19:20.976111+0000","last_peered":"2023-10-09T06:19:20.976111+0000","last_clean":"2023-10-09T06:19:20.976111+0000","last_became_active":"2023-10-09T06:17:45.111092+0000","last_became_peered":"2023-10-09T06:17:45.111092+0000","last_unstale":"2023-10-09T06:19:20.976111+0000","last_undegraded":"2023-10-09T06:19:20.976111+0000","last_fullsized":"2023-10-09T06:19:20.976111+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:05.640906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365821952,"num_objects":5582,"num_object_clones":0,"num_object_copies":16746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5582,"num_write_kb":357248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":18,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.845919+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:19:02.845919+0000","last_peered":"2023-10-09T06:19:02.845919+0000","last_clean":"2023-10-09T06:19:02.845919+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:19:02.845919+0000","last_undegraded":"2023-10-09T06:19:02.845919+0000","last_fullsized":"2023-10-09T06:19:02.845919+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":101,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.850003+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:19:02.850003+0000","last_peered":"2023-10-09T06:19:02.850003+0000","last_clean":"2023-10-09T06:19:02.850003+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:19:02.850003+0000","last_undegraded":"2023-10-09T06:19:02.850003+0000","last_fullsized":"2023-10-09T06:19:02.850003+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"121'5723","reported_seq":5748,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:20.911169+0000","last_change":"2023-10-09T06:17:45.115352+0000","last_active":"2023-10-09T06:19:20.911169+0000","last_peered":"2023-10-09T06:19:20.911169+0000","last_clean":"2023-10-09T06:19:20.911169+0000","last_became_active":"2023-10-09T06:17:45.114984+0000","last_became_peered":"2023-10-09T06:17:45.114984+0000","last_unstale":"2023-10-09T06:19:20.911169+0000","last_undegraded":"2023-10-09T06:19:20.911169+0000","last_fullsized":"2023-10-09T06:19:20.911169+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:13.631395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375062528,"num_objects":5723,"num_object_clones":0,"num_object_copies":17169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5723,"num_write_kb":366272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":18,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.834057+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:19:02.834057+0000","last_peered":"2023-10-09T06:19:02.834057+0000","last_clean":"2023-10-09T06:19:02.834057+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:19:02.834057+0000","last_undegraded":"2023-10-09T06:19:02.834057+0000","last_fullsized":"2023-10-09T06:19:02.834057+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":105,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.835165+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:19:02.835165+0000","last_peered":"2023-10-09T06:19:02.835165+0000","last_clean":"2023-10-09T06:19:02.835165+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:19:02.835165+0000","last_undegraded":"2023-10-09T06:19:02.835165+0000","last_fullsized":"2023-10-09T06:19:02.835165+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"121'5823","reported_seq":5848,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:20.980177+0000","last_change":"2023-10-09T06:17:45.120273+0000","last_active":"2023-10-09T06:19:20.980177+0000","last_peered":"2023-10-09T06:19:20.980177+0000","last_clean":"2023-10-09T06:19:20.980177+0000","last_became_active":"2023-10-09T06:17:45.119964+0000","last_became_peered":"2023-10-09T06:17:45.119964+0000","last_unstale":"2023-10-09T06:19:20.980177+0000","last_undegraded":"2023-10-09T06:19:20.980177+0000","last_fullsized":"2023-10-09T06:19:20.980177+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:22.213850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381616128,"num_objects":5823,"num_object_clones":0,"num_object_copies":17469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5823,"num_write_kb":372672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.835181+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:19:02.835181+0000","last_peered":"2023-10-09T06:19:02.835181+0000","last_clean":"2023-10-09T06:19:02.835181+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:19:02.835181+0000","last_undegraded":"2023-10-09T06:19:02.835181+0000","last_fullsized":"2023-10-09T06:19:02.835181+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.820841+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:19:02.820841+0000","last_peered":"2023-10-09T06:19:02.820841+0000","last_clean":"2023-10-09T06:19:02.820841+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:19:02.820841+0000","last_undegraded":"2023-10-09T06:19:02.820841+0000","last_fullsized":"2023-10-09T06:19:02.820841+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"121'5711","reported_seq":5752,"reported_epoch":121,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:20.979604+0000","last_change":"2023-10-09T06:19:01.121556+0000","last_active":"2023-10-09T06:19:20.979604+0000","last_peered":"2023-10-09T06:19:20.979604+0000","last_clean":"2023-10-09T06:18:58.761034+0000","last_became_active":"2023-10-09T06:19:01.109612+0000","last_became_peered":"2023-10-09T06:19:01.109612+0000","last_unstale":"2023-10-09T06:19:20.979604+0000","last_undegraded":"2023-10-09T06:19:00.825424+0000","last_fullsized":"2023-10-09T06:19:00.781993+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5711,"log_dups_size":0,"ondisk_log_size":5711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:35.553242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374276096,"num_objects":5711,"num_object_clones":0,"num_object_copies":17133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4828,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5711,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":4828},{"shards":"2,5","objects":883}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.827592+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:19:02.827592+0000","last_peered":"2023-10-09T06:19:02.827592+0000","last_clean":"2023-10-09T06:19:02.827592+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:19:02.827592+0000","last_undegraded":"2023-10-09T06:19:02.827592+0000","last_fullsized":"2023-10-09T06:19:02.827592+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":86,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.846029+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:19:02.846029+0000","last_peered":"2023-10-09T06:19:02.846029+0000","last_clean":"2023-10-09T06:19:02.846029+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:19:02.846029+0000","last_undegraded":"2023-10-09T06:19:02.846029+0000","last_fullsized":"2023-10-09T06:19:02.846029+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"122'5771","reported_seq":5797,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:21.510140+0000","last_change":"2023-10-09T06:17:45.120554+0000","last_active":"2023-10-09T06:19:21.510140+0000","last_peered":"2023-10-09T06:19:21.510140+0000","last_clean":"2023-10-09T06:19:21.510140+0000","last_became_active":"2023-10-09T06:17:45.120155+0000","last_became_peered":"2023-10-09T06:17:45.120155+0000","last_unstale":"2023-10-09T06:19:21.510140+0000","last_undegraded":"2023-10-09T06:19:21.510140+0000","last_fullsized":"2023-10-09T06:19:21.510140+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5771,"log_dups_size":0,"ondisk_log_size":5771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:57.941477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208256,"num_objects":5771,"num_object_clones":0,"num_object_copies":17313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5771,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.835278+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:19:02.835278+0000","last_peered":"2023-10-09T06:19:02.835278+0000","last_clean":"2023-10-09T06:19:02.835278+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:19:02.835278+0000","last_undegraded":"2023-10-09T06:19:02.835278+0000","last_fullsized":"2023-10-09T06:19:02.835278+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":101,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.846316+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:19:02.846316+0000","last_peered":"2023-10-09T06:19:02.846316+0000","last_clean":"2023-10-09T06:19:02.846316+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:19:02.846316+0000","last_undegraded":"2023-10-09T06:19:02.846316+0000","last_fullsized":"2023-10-09T06:19:02.846316+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"122'5938","reported_seq":5963,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:24.505865+0000","last_change":"2023-10-09T06:17:45.125233+0000","last_active":"2023-10-09T06:19:24.505865+0000","last_peered":"2023-10-09T06:19:24.505865+0000","last_clean":"2023-10-09T06:19:24.505865+0000","last_became_active":"2023-10-09T06:17:45.124741+0000","last_became_peered":"2023-10-09T06:17:45.124741+0000","last_unstale":"2023-10-09T06:19:24.505865+0000","last_undegraded":"2023-10-09T06:19:24.505865+0000","last_fullsized":"2023-10-09T06:19:24.505865+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389152768,"num_objects":5938,"num_object_clones":0,"num_object_copies":17814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5938,"num_write_kb":380032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":105,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.849801+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:19:02.849801+0000","last_peered":"2023-10-09T06:19:02.849801+0000","last_clean":"2023-10-09T06:19:02.849801+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:19:02.849801+0000","last_undegraded":"2023-10-09T06:19:02.849801+0000","last_fullsized":"2023-10-09T06:19:02.849801+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":86,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.846060+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:19:02.846060+0000","last_peered":"2023-10-09T06:19:02.846060+0000","last_clean":"2023-10-09T06:19:02.846060+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:19:02.846060+0000","last_undegraded":"2023-10-09T06:19:02.846060+0000","last_fullsized":"2023-10-09T06:19:02.846060+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"122'5794","reported_seq":5818,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:24.520323+0000","last_change":"2023-10-09T06:17:45.124190+0000","last_active":"2023-10-09T06:19:24.520323+0000","last_peered":"2023-10-09T06:19:24.520323+0000","last_clean":"2023-10-09T06:19:24.520323+0000","last_became_active":"2023-10-09T06:17:45.123846+0000","last_became_peered":"2023-10-09T06:17:45.123846+0000","last_unstale":"2023-10-09T06:19:24.520323+0000","last_undegraded":"2023-10-09T06:19:24.520323+0000","last_fullsized":"2023-10-09T06:19:24.520323+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379715584,"num_objects":5794,"num_object_clones":0,"num_object_copies":17382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":370816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":110,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.811321+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:19:01.811321+0000","last_peered":"2023-10-09T06:19:01.811321+0000","last_clean":"2023-10-09T06:19:01.811321+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:19:01.811321+0000","last_undegraded":"2023-10-09T06:19:01.811321+0000","last_fullsized":"2023-10-09T06:19:01.811321+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810969+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:19:02.810969+0000","last_peered":"2023-10-09T06:19:02.810969+0000","last_clean":"2023-10-09T06:19:02.810969+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:19:02.810969+0000","last_undegraded":"2023-10-09T06:19:02.810969+0000","last_fullsized":"2023-10-09T06:19:02.810969+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":931,"reported_epoch":121,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:02.810526+0000","last_change":"2023-10-09T06:19:00.832644+0000","last_active":"2023-10-09T06:19:02.810526+0000","last_peered":"2023-10-09T06:19:02.810526+0000","last_clean":"2023-10-09T06:18:54.772534+0000","last_became_active":"2023-10-09T06:19:00.832133+0000","last_became_peered":"2023-10-09T06:19:00.832133+0000","last_unstale":"2023-10-09T06:19:02.810526+0000","last_undegraded":"2023-10-09T06:19:00.785661+0000","last_fullsized":"2023-10-09T06:19:00.784414+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:27.419992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0],"avail_no_missing":["1","0","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.836119+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:19:02.836119+0000","last_peered":"2023-10-09T06:19:02.836119+0000","last_clean":"2023-10-09T06:19:02.836119+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:19:02.836119+0000","last_undegraded":"2023-10-09T06:19:02.836119+0000","last_fullsized":"2023-10-09T06:19:02.836119+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810282+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:19:02.810282+0000","last_peered":"2023-10-09T06:19:02.810282+0000","last_clean":"2023-10-09T06:19:02.810282+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:19:02.810282+0000","last_undegraded":"2023-10-09T06:19:02.810282+0000","last_fullsized":"2023-10-09T06:19:02.810282+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"122'5750","reported_seq":5776,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:22.809756+0000","last_change":"2023-10-09T06:17:45.113729+0000","last_active":"2023-10-09T06:19:22.809756+0000","last_peered":"2023-10-09T06:19:22.809756+0000","last_clean":"2023-10-09T06:19:22.809756+0000","last_became_active":"2023-10-09T06:17:45.113408+0000","last_became_peered":"2023-10-09T06:17:45.113408+0000","last_unstale":"2023-10-09T06:19:22.809756+0000","last_undegraded":"2023-10-09T06:19:22.809756+0000","last_fullsized":"2023-10-09T06:19:22.809756+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:02.025674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376832000,"num_objects":5750,"num_object_clones":0,"num_object_copies":17250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5750,"num_write_kb":368000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"122'5752","reported_seq":5778,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:22.791100+0000","last_change":"2023-10-09T06:17:45.121476+0000","last_active":"2023-10-09T06:19:22.791100+0000","last_peered":"2023-10-09T06:19:22.791100+0000","last_clean":"2023-10-09T06:19:22.791100+0000","last_became_active":"2023-10-09T06:17:45.121043+0000","last_became_peered":"2023-10-09T06:17:45.121043+0000","last_unstale":"2023-10-09T06:19:22.791100+0000","last_undegraded":"2023-10-09T06:19:22.791100+0000","last_fullsized":"2023-10-09T06:19:22.791100+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:16.155506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376963072,"num_objects":5752,"num_object_clones":0,"num_object_copies":17256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5752,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":703,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.836455+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:19:02.836455+0000","last_peered":"2023-10-09T06:19:02.836455+0000","last_clean":"2023-10-09T06:19:02.836455+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:19:02.836455+0000","last_undegraded":"2023-10-09T06:19:02.836455+0000","last_fullsized":"2023-10-09T06:19:02.836455+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":87,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.845704+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:19:02.845704+0000","last_peered":"2023-10-09T06:19:02.845704+0000","last_clean":"2023-10-09T06:19:02.845704+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:19:02.845704+0000","last_undegraded":"2023-10-09T06:19:02.845704+0000","last_fullsized":"2023-10-09T06:19:02.845704+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810781+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:19:02.810781+0000","last_peered":"2023-10-09T06:19:02.810781+0000","last_clean":"2023-10-09T06:19:02.810781+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:19:02.810781+0000","last_undegraded":"2023-10-09T06:19:02.810781+0000","last_fullsized":"2023-10-09T06:19:02.810781+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"121'5563","reported_seq":5604,"reported_epoch":121,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:20.991873+0000","last_change":"2023-10-09T06:19:01.101635+0000","last_active":"2023-10-09T06:19:20.991873+0000","last_peered":"2023-10-09T06:19:20.991873+0000","last_clean":"2023-10-09T06:18:58.773534+0000","last_became_active":"2023-10-09T06:19:01.091438+0000","last_became_peered":"2023-10-09T06:19:01.091438+0000","last_unstale":"2023-10-09T06:19:20.991873+0000","last_undegraded":"2023-10-09T06:19:00.824313+0000","last_fullsized":"2023-10-09T06:19:00.781613+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:22.355712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364576768,"num_objects":5563,"num_object_clones":0,"num_object_copies":16689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":356032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":4618},{"shards":"2,5","objects":945}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":478,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.815923+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:19:01.815923+0000","last_peered":"2023-10-09T06:19:01.815923+0000","last_clean":"2023-10-09T06:19:01.815923+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:19:01.815923+0000","last_undegraded":"2023-10-09T06:19:01.815923+0000","last_fullsized":"2023-10-09T06:19:01.815923+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810446+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:19:02.810446+0000","last_peered":"2023-10-09T06:19:02.810446+0000","last_clean":"2023-10-09T06:19:02.810446+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:19:02.810446+0000","last_undegraded":"2023-10-09T06:19:02.810446+0000","last_fullsized":"2023-10-09T06:19:02.810446+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":97,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810882+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:19:02.810882+0000","last_peered":"2023-10-09T06:19:02.810882+0000","last_clean":"2023-10-09T06:19:02.810882+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:19:02.810882+0000","last_undegraded":"2023-10-09T06:19:02.810882+0000","last_fullsized":"2023-10-09T06:19:02.810882+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":470,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.854626+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:19:02.854626+0000","last_peered":"2023-10-09T06:19:02.854626+0000","last_clean":"2023-10-09T06:19:02.854626+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:19:02.854626+0000","last_undegraded":"2023-10-09T06:19:02.854626+0000","last_fullsized":"2023-10-09T06:19:02.854626+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":318,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.846692+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:19:02.846692+0000","last_peered":"2023-10-09T06:19:02.846692+0000","last_clean":"2023-10-09T06:19:02.846692+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:19:02.846692+0000","last_undegraded":"2023-10-09T06:19:02.846692+0000","last_fullsized":"2023-10-09T06:19:02.846692+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":82,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.811499+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:19:01.811499+0000","last_peered":"2023-10-09T06:19:01.811499+0000","last_clean":"2023-10-09T06:19:01.811499+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:19:01.811499+0000","last_undegraded":"2023-10-09T06:19:01.811499+0000","last_fullsized":"2023-10-09T06:19:01.811499+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":90,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.834754+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:19:02.834754+0000","last_peered":"2023-10-09T06:19:02.834754+0000","last_clean":"2023-10-09T06:19:02.834754+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:19:02.834754+0000","last_undegraded":"2023-10-09T06:19:02.834754+0000","last_fullsized":"2023-10-09T06:19:02.834754+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"122'5869","reported_seq":5899,"reported_epoch":122,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T06:19:22.806092+0000","last_change":"2023-10-09T06:19:01.117941+0000","last_active":"2023-10-09T06:19:22.806092+0000","last_peered":"2023-10-09T06:19:22.806092+0000","last_clean":"2023-10-09T06:18:58.712907+0000","last_became_active":"2023-10-09T06:19:01.104546+0000","last_became_peered":"2023-10-09T06:19:01.104546+0000","last_unstale":"2023-10-09T06:19:22.806092+0000","last_undegraded":"2023-10-09T06:19:22.806092+0000","last_fullsized":"2023-10-09T06:19:22.806092+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:40.858512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384630784,"num_objects":5869,"num_object_clones":0,"num_object_copies":17607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4827,"num_objects_unfound":0,"num_objects_dirty":5869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5869,"num_write_kb":375616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[0,2,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":1042}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"122'5914","reported_seq":5939,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:24.481964+0000","last_change":"2023-10-09T06:17:45.120249+0000","last_active":"2023-10-09T06:19:24.481964+0000","last_peered":"2023-10-09T06:19:24.481964+0000","last_clean":"2023-10-09T06:19:24.481964+0000","last_became_active":"2023-10-09T06:17:45.119883+0000","last_became_peered":"2023-10-09T06:17:45.119883+0000","last_unstale":"2023-10-09T06:19:24.481964+0000","last_undegraded":"2023-10-09T06:19:24.481964+0000","last_fullsized":"2023-10-09T06:19:24.481964+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387579904,"num_objects":5914,"num_object_clones":0,"num_object_copies":17742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5914,"num_write_kb":378496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":43,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.828332+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:19:02.828332+0000","last_peered":"2023-10-09T06:19:02.828332+0000","last_clean":"2023-10-09T06:19:02.828332+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:19:02.828332+0000","last_undegraded":"2023-10-09T06:19:02.828332+0000","last_fullsized":"2023-10-09T06:19:02.828332+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.820874+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:19:02.820874+0000","last_peered":"2023-10-09T06:19:02.820874+0000","last_clean":"2023-10-09T06:19:02.820874+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:19:02.820874+0000","last_undegraded":"2023-10-09T06:19:02.820874+0000","last_fullsized":"2023-10-09T06:19:02.820874+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":86,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.821892+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:19:02.821892+0000","last_peered":"2023-10-09T06:19:02.821892+0000","last_clean":"2023-10-09T06:19:02.821892+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:19:02.821892+0000","last_undegraded":"2023-10-09T06:19:02.821892+0000","last_fullsized":"2023-10-09T06:19:02.821892+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":32,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.833989+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:19:02.833989+0000","last_peered":"2023-10-09T06:19:02.833989+0000","last_clean":"2023-10-09T06:19:02.833989+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:19:02.833989+0000","last_undegraded":"2023-10-09T06:19:02.833989+0000","last_fullsized":"2023-10-09T06:19:02.833989+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810701+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:19:02.810701+0000","last_peered":"2023-10-09T06:19:02.810701+0000","last_clean":"2023-10-09T06:19:02.810701+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:19:02.810701+0000","last_undegraded":"2023-10-09T06:19:02.810701+0000","last_fullsized":"2023-10-09T06:19:02.810701+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":83,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.821526+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:19:02.821526+0000","last_peered":"2023-10-09T06:19:02.821526+0000","last_clean":"2023-10-09T06:19:02.821526+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:19:02.821526+0000","last_undegraded":"2023-10-09T06:19:02.821526+0000","last_fullsized":"2023-10-09T06:19:02.821526+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"122'5828","reported_seq":5853,"reported_epoch":122,"state":"active+clean","last_fresh":"2023-10-09T06:19:24.504309+0000","last_change":"2023-10-09T06:17:45.120130+0000","last_active":"2023-10-09T06:19:24.504309+0000","last_peered":"2023-10-09T06:19:24.504309+0000","last_clean":"2023-10-09T06:19:24.504309+0000","last_became_active":"2023-10-09T06:17:45.119743+0000","last_became_peered":"2023-10-09T06:17:45.119743+0000","last_unstale":"2023-10-09T06:19:24.504309+0000","last_undegraded":"2023-10-09T06:19:24.504309+0000","last_fullsized":"2023-10-09T06:19:24.504309+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381943808,"num_objects":5828,"num_object_clones":0,"num_object_copies":17484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":372992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":116,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.812679+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:19:01.812679+0000","last_peered":"2023-10-09T06:19:01.812679+0000","last_clean":"2023-10-09T06:19:01.812679+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:19:01.812679+0000","last_undegraded":"2023-10-09T06:19:01.812679+0000","last_fullsized":"2023-10-09T06:19:01.812679+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":424,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.810294+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:19:02.810294+0000","last_peered":"2023-10-09T06:19:02.810294+0000","last_clean":"2023-10-09T06:19:02.810294+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:19:02.810294+0000","last_undegraded":"2023-10-09T06:19:02.810294+0000","last_fullsized":"2023-10-09T06:19:02.810294+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":114,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.811284+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:19:01.811284+0000","last_peered":"2023-10-09T06:19:01.811284+0000","last_clean":"2023-10-09T06:19:01.811284+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:19:01.811284+0000","last_undegraded":"2023-10-09T06:19:01.811284+0000","last_fullsized":"2023-10-09T06:19:01.811284+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"122'5824","reported_seq":6614,"reported_epoch":122,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:19:24.552967+0000","last_change":"2023-10-09T06:19:01.302678+0000","last_active":"2023-10-09T06:19:24.552967+0000","last_peered":"2023-10-09T06:19:24.552967+0000","last_clean":"2023-10-09T06:18:58.760993+0000","last_became_active":"2023-10-09T06:19:01.088682+0000","last_became_peered":"2023-10-09T06:19:01.088682+0000","last_unstale":"2023-10-09T06:19:24.552967+0000","last_undegraded":"2023-10-09T06:19:24.552967+0000","last_fullsized":"2023-10-09T06:19:00.792897+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381681664,"num_objects":5824,"num_object_clones":0,"num_object_copies":17472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4412,"num_objects_unfound":0,"num_objects_dirty":5824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5824,"num_write_kb":372736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":375,"num_bytes_recovered":24576000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":4412},{"shards":"3,4","objects":1412}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":787,"reported_epoch":121,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:02.834122+0000","last_change":"2023-10-09T06:19:00.801734+0000","last_active":"2023-10-09T06:19:02.834122+0000","last_peered":"2023-10-09T06:19:02.834122+0000","last_clean":"2023-10-09T06:18:54.789583+0000","last_became_active":"2023-10-09T06:19:00.801207+0000","last_became_peered":"2023-10-09T06:19:00.801207+0000","last_unstale":"2023-10-09T06:19:02.834122+0000","last_undegraded":"2023-10-09T06:19:00.787699+0000","last_fullsized":"2023-10-09T06:19:00.787091+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":75,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.821824+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:19:02.821824+0000","last_peered":"2023-10-09T06:19:02.821824+0000","last_clean":"2023-10-09T06:19:02.821824+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:19:02.821824+0000","last_undegraded":"2023-10-09T06:19:02.821824+0000","last_fullsized":"2023-10-09T06:19:02.821824+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.849880+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:19:02.849880+0000","last_peered":"2023-10-09T06:19:02.849880+0000","last_clean":"2023-10-09T06:19:02.849880+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:19:02.849880+0000","last_undegraded":"2023-10-09T06:19:02.849880+0000","last_fullsized":"2023-10-09T06:19:02.849880+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"121'5820","reported_seq":5845,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:20.717454+0000","last_change":"2023-10-09T06:17:45.119555+0000","last_active":"2023-10-09T06:19:20.717454+0000","last_peered":"2023-10-09T06:19:20.717454+0000","last_clean":"2023-10-09T06:19:20.717454+0000","last_became_active":"2023-10-09T06:17:45.119179+0000","last_became_peered":"2023-10-09T06:17:45.119179+0000","last_unstale":"2023-10-09T06:19:20.717454+0000","last_undegraded":"2023-10-09T06:19:20.717454+0000","last_fullsized":"2023-10-09T06:19:20.717454+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:15.093934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381419520,"num_objects":5820,"num_object_clones":0,"num_object_copies":17460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5820,"num_write_kb":372480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":78,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.821356+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:19:02.821356+0000","last_peered":"2023-10-09T06:19:02.821356+0000","last_clean":"2023-10-09T06:19:02.821356+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:19:02.821356+0000","last_undegraded":"2023-10-09T06:19:02.821356+0000","last_fullsized":"2023-10-09T06:19:02.821356+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":74,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.815557+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:19:02.815557+0000","last_peered":"2023-10-09T06:19:02.815557+0000","last_clean":"2023-10-09T06:19:02.815557+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:19:02.815557+0000","last_undegraded":"2023-10-09T06:19:02.815557+0000","last_fullsized":"2023-10-09T06:19:02.815557+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":89,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.835329+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:19:02.835329+0000","last_peered":"2023-10-09T06:19:02.835329+0000","last_clean":"2023-10-09T06:19:02.835329+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:19:02.835329+0000","last_undegraded":"2023-10-09T06:19:02.835329+0000","last_fullsized":"2023-10-09T06:19:02.835329+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.845944+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:19:02.845944+0000","last_peered":"2023-10-09T06:19:02.845944+0000","last_clean":"2023-10-09T06:19:02.845944+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:19:02.845944+0000","last_undegraded":"2023-10-09T06:19:02.845944+0000","last_fullsized":"2023-10-09T06:19:02.845944+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":74,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.836205+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:19:02.836205+0000","last_peered":"2023-10-09T06:19:02.836205+0000","last_clean":"2023-10-09T06:19:02.836205+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:19:02.836205+0000","last_undegraded":"2023-10-09T06:19:02.836205+0000","last_fullsized":"2023-10-09T06:19:02.836205+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.851301+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:19:02.851301+0000","last_peered":"2023-10-09T06:19:02.851301+0000","last_clean":"2023-10-09T06:19:02.851301+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:19:02.851301+0000","last_undegraded":"2023-10-09T06:19:02.851301+0000","last_fullsized":"2023-10-09T06:19:02.851301+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":97,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.812859+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:19:01.812859+0000","last_peered":"2023-10-09T06:19:01.812859+0000","last_clean":"2023-10-09T06:19:01.812859+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:19:01.812859+0000","last_undegraded":"2023-10-09T06:19:01.812859+0000","last_fullsized":"2023-10-09T06:19:01.812859+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":109,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-09T06:19:01.812280+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:19:01.812280+0000","last_peered":"2023-10-09T06:19:01.812280+0000","last_clean":"2023-10-09T06:19:01.812280+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:19:01.812280+0000","last_undegraded":"2023-10-09T06:19:01.812280+0000","last_fullsized":"2023-10-09T06:19:01.812280+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":75,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.822279+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:19:02.822279+0000","last_peered":"2023-10-09T06:19:02.822279+0000","last_clean":"2023-10-09T06:19:02.822279+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:19:02.822279+0000","last_undegraded":"2023-10-09T06:19:02.822279+0000","last_fullsized":"2023-10-09T06:19:02.822279+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":86,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-09T06:19:02.852929+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:19:02.852929+0000","last_peered":"2023-10-09T06:19:02.852929+0000","last_clean":"2023-10-09T06:19:02.852929+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:19:02.852929+0000","last_undegraded":"2023-10-09T06:19:02.852929+0000","last_fullsized":"2023-10-09T06:19:02.852929+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6058475520,"num_objects":92445,"num_object_clones":0,"num_object_copies":277335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9446,"num_objects_misplaced":14091,"num_objects_unfound":0,"num_objects_dirty":92445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92445,"num_write_kb":5916480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":375,"num_bytes_recovered":24576000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18447990784,"data_stored":18447990784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92445,"ondisk_log_size":92445,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430457,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":663040,"kb_used_data":357624,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93082112,"statfs":{"total":95995035648,"available":95316082688,"internally_reserved":0,"allocated":366206976,"data_stored":365693073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.90200000000000002,"15min":0.874},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.7549999999999999,"5min":1.7549999999999999,"15min":1.7549999999999999},"last":1.034},{"interface":"front","average":{"1min":0.876,"5min":0.89000000000000001,"15min":0.86399999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.66100000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.93500000000000005,"15min":0.88100000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4079999999999999,"5min":3.4079999999999999,"15min":3.4079999999999999},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.99399999999999999,"15min":0.93700000000000006},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.4590000000000001,"5min":3.4590000000000001,"15min":3.4590000000000001},"last":0.996}]},{"osd":2,"last update":"Mon Oct 9 06:19:20 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":0.96199999999999997,"15min":0.94599999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.327,"5min":3.327,"15min":3.327},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.0800000000000001,"15min":1.022},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.0170000000000003,"5min":6.0170000000000003,"15min":6.0170000000000003},"last":0.90100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.95699999999999996,"15min":0.997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.1269999999999998,"5min":2.1269999999999998,"15min":2.1269999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0980000000000001,"15min":1.133},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":3.806,"5min":3.806,"15min":3.806},"last":0.95399999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:18:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.872,"15min":0.90800000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.92300000000000004,"15min":0.93500000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3160000000000001,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.442}]},{"osd":5,"last update":"Mon Oct 9 06:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.129,"15min":1.0289999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":3.7599999999999998,"5min":3.7599999999999998,"15min":3.7599999999999998},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.046,"15min":1.0649999999999999},"min":{"1min":0.48799999999999999,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":3.8380000000000001,"5min":3.8380000000000001,"15min":3.8380000000000001},"last":0.74299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:18:35 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0429999999999999,"15min":1.083},"min":{"1min":0.52200000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.744,"5min":1.744,"15min":1.744},"last":0.87},{"interface":"front","average":{"1min":1.125,"5min":1.0880000000000001,"15min":1.0680000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":0.58599999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626689,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1806320,"kb_used_data":1500904,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91938832,"statfs":{"total":95995035648,"available":94145363968,"internally_reserved":0,"allocated":1536925696,"data_stored":1536427420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.90700000000000003,"15min":0.88700000000000001},"min":{"1min":0.50700000000000001,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.4259999999999999,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.877,"5min":0.89300000000000002,"15min":0.92900000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.88600000000000001,"15min":0.86699999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.3519999999999999,"5min":2.3519999999999999,"15min":2.3519999999999999},"last":5.8840000000000003},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.88900000000000001,"15min":0.89000000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":5.8129999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:19:19 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.94099999999999995,"15min":0.96199999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4220000000000002,"5min":2.4220000000000002,"15min":2.4220000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.874,"5min":1.046,"15min":0.97099999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.625,"5min":8.2769999999999992,"15min":8.2769999999999992},"last":3.1840000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0009999999999999,"15min":1.0469999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.98199999999999998,"15min":1.002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.3250000000000002,"5min":2.3250000000000002,"15min":2.3250000000000002},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.048,"15min":0.94799999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":4.7750000000000004,"5min":4.7750000000000004,"15min":4.7750000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.286,"5min":1.0840000000000001,"15min":0.995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.738,"5min":2.738,"15min":2.738},"last":0.33900000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:18:55 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.075,"15min":1.0129999999999999},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":2.5920000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.395,"5min":1.135,"15min":1.0660000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":4.085,"5min":4.085,"15min":4.085},"last":1.3540000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:18:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.1180000000000001,"15min":1.075},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.47799999999999998},"max":{"1min":2.0449999999999999,"5min":2.0449999999999999,"15min":2.0449999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.155,"15min":1.115},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.496},"max":{"1min":1.909,"5min":1.909,"15min":1.909},"last":1.044}]}]},{"osd":1,"up_from":93,"seq":399431958598,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2279112,"kb_used_data":2203136,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":91466040,"statfs":{"total":95995035648,"available":93661224960,"internally_reserved":0,"allocated":2256011264,"data_stored":2255428689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":77783440},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":0.76400000000000001,"15min":0.73799999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.2069999999999999,"5min":4.2069999999999999,"15min":4.2069999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.306,"5min":0.80600000000000005,"15min":0.72799999999999998},"min":{"1min":0.433,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.3769999999999998,"5min":4.3769999999999998,"15min":4.3769999999999998},"last":3.3559999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:18:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.78800000000000003,"15min":0.71399999999999997},"min":{"1min":0.371,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":7.6289999999999996,"5min":7.6289999999999996,"15min":7.6289999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.1579999999999999,"5min":0.88800000000000001,"15min":0.84999999999999998},"min":{"1min":0.53300000000000003,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":7.7850000000000001,"5min":7.7850000000000001,"15min":7.7850000000000001},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":1.046,"15min":0.90700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":16.390000000000001,"5min":16.390000000000001,"15min":16.390000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.641,"5min":0.91900000000000004,"15min":0.82799999999999996},"min":{"1min":0.378,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":16.555,"5min":16.555,"15min":16.555},"last":0.88100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0309999999999999,"15min":0.98599999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.9790000000000001,"5min":4.9790000000000001,"15min":4.9790000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.95799999999999996,"15min":0.99399999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":2.2450000000000001,"5min":2.2450000000000001,"15min":2.2450000000000001},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.085,"15min":1.0509999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":5.3630000000000004,"5min":5.3630000000000004,"15min":5.3630000000000004},"last":3.4060000000000001},{"interface":"front","average":{"1min":1.22,"5min":1.1180000000000001,"15min":1.0600000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":5.0170000000000003,"5min":5.0170000000000003,"15min":5.0170000000000003},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.1080000000000001,"15min":1.0880000000000001},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":4.29,"5min":4.29,"15min":4.29},"last":3.4409999999999998},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.145,"15min":1.133},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.0529999999999999,"5min":5.0529999999999999,"15min":5.0529999999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.131,"15min":1.125},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":5.0330000000000004,"5min":5.0330000000000004,"15min":5.0330000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.0609999999999999,"15min":1.1419999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.984,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.55200000000000005}]}]},{"osd":0,"up_from":88,"seq":377957122124,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4041508,"kb_used_data":3933532,"kb_used_omap":8,"kb_used_meta":107959,"kb_avail":89703644,"statfs":{"total":95995035648,"available":91856531456,"internally_reserved":0,"allocated":4027936768,"data_stored":4027334808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":110550272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,0,0,0,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:18:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":0.81599999999999995,"15min":0.66600000000000004},"min":{"1min":0.44900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.835,"5min":5.835,"15min":5.835},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.274,"5min":0.86699999999999999,"15min":0.76100000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.243,"5min":10.243,"15min":10.243},"last":0.61299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:19:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":0.86799999999999999,"15min":0.72599999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":8.1359999999999992,"5min":8.1359999999999992,"15min":8.1359999999999992},"last":3.552},{"interface":"front","average":{"1min":2.3050000000000002,"5min":1.0640000000000001,"15min":0.83599999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.896000000000001,"5min":14.896000000000001,"15min":14.896000000000001},"last":12.19}]},{"osd":3,"last update":"Mon Oct 9 06:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":0.85999999999999999,"15min":0.79400000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.0620000000000003,"5min":7.0620000000000003,"15min":7.0620000000000003},"last":0.307},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.752,"15min":0.76600000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.3109999999999999,"5min":2.3109999999999999,"15min":2.3109999999999999},"last":1.6910000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.018,"15min":0.95999999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.3209999999999997,"5min":6.3209999999999997,"15min":6.3209999999999997},"last":1.3520000000000001},{"interface":"front","average":{"1min":1.3859999999999999,"5min":1.0569999999999999,"15min":1.022},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":8.1129999999999995,"5min":8.1129999999999995,"15min":8.1129999999999995},"last":1.415}]},{"osd":5,"last update":"Mon Oct 9 06:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.071,"15min":1.0549999999999999},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":8.0760000000000005,"5min":8.0760000000000005,"15min":8.0760000000000005},"last":1.373},{"interface":"front","average":{"1min":1.167,"5min":1.0509999999999999,"15min":1.0609999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.6040000000000001,"5min":2.6040000000000001,"15min":2.6040000000000001},"last":0.96599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.042,"15min":1.0720000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":2.649,"5min":2.649,"15min":2.649},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.306,"5min":1.0649999999999999,"15min":1.03},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":4.4000000000000004,"5min":4.4000000000000004,"15min":4.4000000000000004},"last":1.3029999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:18:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.1200000000000001,"15min":1.1459999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":2.54,"5min":2.54,"15min":2.54},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.147,"15min":1.1299999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.4359999999999999,"5min":4.4359999999999999,"15min":4.4359999999999999},"last":1.431}]}]},{"osd":2,"up_from":98,"seq":420906795072,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2264568,"kb_used_data":2192752,"kb_used_omap":11,"kb_used_meta":71796,"kb_avail":91480584,"statfs":{"total":95995035648,"available":93676118016,"internally_reserved":0,"allocated":2245378048,"data_stored":2244806926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":73520062},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.71499999999999997,"15min":0.69899999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.6600000000000001,"5min":3.6600000000000001,"15min":3.6600000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.73099999999999998,"15min":0.67400000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.9169999999999998,"5min":2.9169999999999998,"15min":2.9169999999999998},"last":2.7869999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.76400000000000001,"15min":0.72199999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.6989999999999998,"5min":3.6989999999999998,"15min":3.6989999999999998},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.874,"5min":0.77000000000000002,"15min":0.71999999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.0339999999999998,"5min":3.0339999999999998,"15min":3.0339999999999998},"last":3.5790000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":0.80600000000000005,"15min":0.754},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.7669999999999999,"5min":3.7669999999999999,"15min":3.7669999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.68000000000000005,"15min":0.69399999999999995},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.0840000000000001,"5min":1.256,"15min":1.256},"last":0.66200000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.0049999999999999,"15min":0.96899999999999997},"min":{"1min":0.62,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":3.3969999999999998,"5min":3.3969999999999998,"15min":3.3969999999999998},"last":1.3879999999999999},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.93899999999999995,"15min":0.92700000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.0980000000000001,"5min":1.488,"15min":1.488},"last":6.1699999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.02,"15min":1.02},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":2.9399999999999999,"5min":2.9399999999999999,"15min":2.9399999999999999},"last":3.407},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.94899999999999995,"15min":0.94499999999999995},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.63200000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.996,"15min":1.0009999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":1.4590000000000001},{"interface":"front","average":{"1min":1.351,"5min":1.0860000000000001,"15min":1.022},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":3.9289999999999998,"5min":3.9289999999999998,"15min":3.9289999999999998},"last":0.73999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.1479999999999999,"15min":1.1140000000000001},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":3.8839999999999999,"5min":3.8839999999999999,"15min":3.8839999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.083,"5min":1.1020000000000001,"15min":1.099},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":2.7290000000000001,"5min":2.7290000000000001,"15min":2.7290000000000001},"last":1.3520000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631546,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155236,"kb_used_data":1105692,"kb_used_omap":3,"kb_used_meta":49532,"kb_avail":92589916,"statfs":{"total":95995035648,"available":94812073984,"internally_reserved":0,"allocated":1132228608,"data_stored":1131675497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":50721313},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":0.745,"15min":0.69299999999999995},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":4.016,"5min":4.016,"15min":4.016},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.052,"5min":0.74099999999999999,"15min":0.67900000000000005},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.5289999999999999,"5min":5.5289999999999999,"15min":5.5289999999999999},"last":1.1140000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.70099999999999996,"15min":0.70899999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.984,"5min":1.984,"15min":1.984},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.044,"5min":0.83299999999999996,"15min":0.78300000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.4859999999999998,"5min":6.4859999999999998,"15min":6.4859999999999998},"last":3.4359999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":0.878,"15min":0.83799999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5949999999999998,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.74299999999999999,"15min":0.73799999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":0.69199999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.91800000000000004,"15min":0.92000000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1150000000000002,"5min":2.1150000000000002,"15min":2.1150000000000002},"last":2.855},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.96599999999999997,"15min":0.98799999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":2.2709999999999999,"5min":2.2709999999999999,"15min":2.2709999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.98899999999999999,"15min":1.026},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.3759999999999999,"5min":2.3759999999999999,"15min":2.3759999999999999},"last":4.4619999999999997},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.0169999999999999,"15min":1.0629999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.1949999999999998,"5min":2.1949999999999998,"15min":2.1949999999999998},"last":1.2090000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.079,"15min":1.105},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.042,"15min":1.0820000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.456,"5min":1.577,"15min":1.577},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:18:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.042,"15min":1.0549999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":2.1560000000000001,"5min":2.1560000000000001,"15min":2.1560000000000001},"last":1.171},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.1020000000000001,"15min":1.1240000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":2.0779999999999998,"5min":2.0779999999999998,"15min":2.0779999999999998},"last":0.68100000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019155,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3626296,"kb_used_data":3316784,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90118856,"statfs":{"total":95995035648,"available":92281708544,"internally_reserved":0,"allocated":3396386816,"data_stored":3395880491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.6799999999999999,"15min":1.202},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.6340000000000003,"5min":40.439999999999998,"15min":40.439999999999998},"last":0.77500000000000002},{"interface":"front","average":{"1min":2.593,"5min":1.746,"15min":1.2689999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":15.364000000000001,"5min":40.494999999999997,"15min":40.494999999999997},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:18:51 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.0389999999999999,"15min":0.98499999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.3090000000000002,"5min":4.3090000000000002,"15min":4.3090000000000002},"last":1.71},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.117,"15min":1.03},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.0170000000000003,"5min":7.0170000000000003,"15min":7.0170000000000003},"last":1.8009999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.1159999999999997,"5min":1.6160000000000001,"15min":1.1439999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":27.134,"5min":27.134,"15min":27.134},"last":0.67100000000000004},{"interface":"front","average":{"1min":4.0750000000000002,"5min":1.623,"15min":1.129},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":27.082000000000001,"5min":27.082000000000001,"15min":27.082000000000001},"last":0.60699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:18:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":1.361,"15min":1.139},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":36.154000000000003,"5min":36.154000000000003,"15min":36.154000000000003},"last":0.747},{"interface":"front","average":{"1min":2.653,"5min":1.3700000000000001,"15min":1.1419999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":36.064,"5min":36.064,"15min":36.064},"last":0.93400000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:19:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.99099999999999999,"15min":0.91100000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.165,"15min":1.02},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":6.702,"5min":6.702,"15min":6.702},"last":0.90500000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:18:44 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.0760000000000001,"15min":1.026},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.7919999999999998,"5min":2.7919999999999998,"15min":2.7919999999999998},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.0820000000000001,"15min":1.004},"min":{"1min":0.437,"5min":0.437,"15min":0.436},"max":{"1min":3.3479999999999999,"5min":3.3479999999999999,"15min":3.3479999999999999},"last":0.85099999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:18:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.1200000000000001,"15min":1.0680000000000001},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52200000000000002},"max":{"1min":2.9510000000000001,"5min":2.9510000000000001,"15min":2.9510000000000001},"last":0.81499999999999995},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.1080000000000001,"15min":1.0629999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.706,"5min":3.706,"15min":3.706},"last":0.98599999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822922,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3668452,"kb_used_data":3358940,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90076700,"statfs":{"total":95995035648,"available":92238540800,"internally_reserved":0,"allocated":3439554560,"data_stored":3439063848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.077,"15min":0.99299999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.629,"5min":11.629,"15min":11.629},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.05,"15min":0.94399999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.7089999999999996,"5min":9.7089999999999996,"15min":9.7089999999999996},"last":4.2510000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":0.95499999999999996,"15min":0.94799999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.1030000000000002,"5min":3.1030000000000002,"15min":3.1030000000000002},"last":0.498},{"interface":"front","average":{"1min":1.1539999999999999,"5min":0.96199999999999997,"15min":0.92300000000000004},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.9769999999999999,"5min":3.9769999999999999,"15min":3.9769999999999999},"last":1.262}]},{"osd":2,"last update":"Mon Oct 9 06:19:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.087,"15min":0.97199999999999998},"min":{"1min":0.442,"5min":0.307,"15min":0.307},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":1.0620000000000001},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.3029999999999999,"15min":1.119},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":5.9210000000000003,"5min":5.9210000000000003,"15min":5.9210000000000003},"last":0.874}]},{"osd":3,"last update":"Mon Oct 9 06:18:54 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.0469999999999999,"15min":1.0189999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.4609999999999999,"5min":3.4609999999999999,"15min":3.4609999999999999},"last":1.097},{"interface":"front","average":{"1min":1.01,"5min":1.0409999999999999,"15min":1.0429999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.5600000000000001,"5min":2.5600000000000001,"15min":2.5600000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 06:19:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.0309999999999999,"15min":0.93600000000000005},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.42499999999999999},"max":{"1min":8.2040000000000006,"5min":8.2040000000000006,"15min":8.2040000000000006},"last":8.7940000000000005},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.05,"15min":1},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":3.3929999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.98299999999999998,"15min":0.98599999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":2.1709999999999998,"5min":2.1709999999999998,"15min":2.1709999999999998},"last":1.7689999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.115,"15min":1.0880000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":3.472,"5min":3.472,"15min":3.472},"last":1.127}]},{"osd":7,"last update":"Mon Oct 9 06:18:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.149,"15min":1.107},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":2.2509999999999999},"last":3.4239999999999999},{"interface":"front","average":{"1min":1.323,"5min":1.1759999999999999,"15min":1.1379999999999999},"min":{"1min":0.51100000000000001,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":3.75,"5min":3.75,"15min":3.75},"last":0.84099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4036362240,"data_stored":4036362240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2260402176,"data_stored":2260402176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2241003520,"data_stored":2241003520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1133182976,"data_stored":1133182976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3397844992,"data_stored":3397844992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3456237568,"data_stored":3456237568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1543307264,"data_stored":1543307264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":379650048,"data_stored":379650048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:19:26.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-10-09T06:19:26.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:26.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:26.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:26.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:26.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:26.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:26.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:27.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:27.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:27.104 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:27.104 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:27.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 95904 95888 75.8513 43.875 0.0264974 0.0131767 2023-10-09T06:19:27.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:27.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:27.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:27.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:19:27.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:27.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:27.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:27.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:27.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:27.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:27.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:27.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:27.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:27 smithi125 bash[77299]: cluster 2023-10-09T06:19:26.686609+0000 mgr.y (mgr.25723) 425 : cluster 0 pgmap v375: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 9469/282963 objects degraded (3.346%); 14091/282963 objects misplaced (4.980%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:27.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:27 smithi125 bash[80002]: cluster 2023-10-09T06:19:26.686609+0000 mgr.y (mgr.25723) 425 : cluster 0 pgmap v375: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 9469/282963 objects degraded (3.346%); 14091/282963 objects misplaced (4.980%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:27.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:27.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:28.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:28.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:28.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:28.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:19:28.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:28.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:27 smithi155 bash[59495]: cluster 2023-10-09T06:19:26.686609+0000 mgr.y (mgr.25723) 425 : cluster 0 pgmap v375: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 9469/282963 objects degraded (3.346%); 14091/282963 objects misplaced (4.980%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:28.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:19:28.172209+0000 min lat: 0.00139442 max lat: 2.36825 avg lat: 0.0132432 2023-10-09T06:19:28.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:19:28.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 96634 96618 75.4735 45.625 0.021889 0.0132432 2023-10-09T06:19:28.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:28.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:28.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:28.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:28.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:28.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:28.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:28.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:28.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:28.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:28.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:28.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:28 smithi125 bash[77299]: cluster 2023-10-09T06:19:28.616496+0000 mon.a (mon.0) 567 : cluster 3 Health check update: Degraded data redundancy: 9469/282963 objects degraded (3.346%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:28.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:28 smithi125 bash[80002]: cluster 2023-10-09T06:19:28.616496+0000 mon.a (mon.0) 567 : cluster 3 Health check update: Degraded data redundancy: 9469/282963 objects degraded (3.346%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:28.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:29.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:29.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:29.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:28 smithi155 bash[59495]: cluster 2023-10-09T06:19:28.616496+0000 mon.a (mon.0) 567 : cluster 3 Health check update: Degraded data redundancy: 9469/282963 objects degraded (3.346%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:29.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:29.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 97373 97357 75.1119 46.1875 0.0116957 0.0133072 2023-10-09T06:19:29.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:29.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:29.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:29.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:29.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:29.759 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:29 smithi155 bash[59495]: cluster 2023-10-09T06:19:28.687250+0000 mgr.y (mgr.25723) 426 : cluster 0 pgmap v376: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 9469/285189 objects degraded (3.320%); 14091/285189 objects misplaced (4.941%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:29.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:29.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:29.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:29.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:29.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:30.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:30.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:30.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:30.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:29 smithi125 bash[77299]: cluster 2023-10-09T06:19:28.687250+0000 mgr.y (mgr.25723) 426 : cluster 0 pgmap v376: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 9469/285189 objects degraded (3.320%); 14091/285189 objects misplaced (4.941%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:30.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:29 smithi125 bash[80002]: cluster 2023-10-09T06:19:28.687250+0000 mgr.y (mgr.25723) 426 : cluster 0 pgmap v376: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 707 op/s; 9469/285189 objects degraded (3.320%); 14091/285189 objects misplaced (4.941%); 998 KiB/s, 15 objects/s recovering 2023-10-09T06:19:30.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 15 98066 98051 74.7223 43.375 0.0404212 0.0133758 2023-10-09T06:19:30.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:30.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:30.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:30.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:30.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:30.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:30.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:30.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:30.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:30.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:30.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:30.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:31.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:31.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:31.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 98734 98718 74.3243 41.6875 0.0269346 0.0134479 2023-10-09T06:19:31.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:31.299 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:31.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:31.418 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:31] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-10-09T06:19:31.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:31.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:31.684 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:31.685 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:31.688 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:31.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:31.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:31.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:31.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:31.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:31.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:31.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:31.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: cluster 2023-10-09T06:19:30.688230+0000 mgr.y (mgr.25723) 427 : cluster 0 pgmap v377: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 9469/288651 objects degraded (3.280%); 14012/288651 objects misplaced (4.854%); 1004 KiB/s, 15 objects/s recovering 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: audit 2023-10-09T06:19:31.060717+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: audit 2023-10-09T06:19:31.074735+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: audit 2023-10-09T06:19:31.707885+0000 mon.a (mon.0) 570 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: audit 2023-10-09T06:19:31.709709+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:32.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:32 smithi155 bash[59495]: audit 2023-10-09T06:19:31.726384+0000 mon.a (mon.0) 572 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:32.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:32.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 99427 99411 73.9551 43.3125 0.0109285 0.0135143 2023-10-09T06:19:32.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:32.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: cluster 2023-10-09T06:19:30.688230+0000 mgr.y (mgr.25723) 427 : cluster 0 pgmap v377: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 9469/288651 objects degraded (3.280%); 14012/288651 objects misplaced (4.854%); 1004 KiB/s, 15 objects/s recovering 2023-10-09T06:19:32.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: audit 2023-10-09T06:19:31.060717+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: audit 2023-10-09T06:19:31.074735+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: audit 2023-10-09T06:19:31.707885+0000 mon.a (mon.0) 570 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: audit 2023-10-09T06:19:31.709709+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[77299]: audit 2023-10-09T06:19:31.726384+0000 mon.a (mon.0) 572 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: cluster 2023-10-09T06:19:30.688230+0000 mgr.y (mgr.25723) 427 : cluster 0 pgmap v377: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 9469/288651 objects degraded (3.280%); 14012/288651 objects misplaced (4.854%); 1004 KiB/s, 15 objects/s recovering 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: audit 2023-10-09T06:19:31.060717+0000 mon.a (mon.0) 568 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: audit 2023-10-09T06:19:31.074735+0000 mon.a (mon.0) 569 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: audit 2023-10-09T06:19:31.707885+0000 mon.a (mon.0) 570 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:32.232 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: audit 2023-10-09T06:19:31.709709+0000 mon.a (mon.0) 571 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:32.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:32 smithi125 bash[80002]: audit 2023-10-09T06:19:31.726384+0000 mon.a (mon.0) 572 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:32.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:32.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:32.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:32.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:32.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:32.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:32.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:32.701 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:32.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:32.804 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:32.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:32.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:32.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:33.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:33.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 100139 100123 73.6085 44.5 0.0210971 0.013579 2023-10-09T06:19:33.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:33.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:33.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:33.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:33.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:33.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:33.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.622 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:33.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:33.768 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:33.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:33.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:33.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:33.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:34.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:34.029 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:34.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:34.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:34 smithi155 bash[59495]: cluster 2023-10-09T06:19:32.688932+0000 mgr.y (mgr.25723) 428 : cluster 0 pgmap v378: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 9469/293856 objects degraded (3.222%); 14012/293856 objects misplaced (4.768%); 889 KiB/s, 13 objects/s recovering 2023-10-09T06:19:34.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[77299]: cluster 2023-10-09T06:19:32.688932+0000 mgr.y (mgr.25723) 428 : cluster 0 pgmap v378: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 9469/293856 objects degraded (3.222%); 14012/293856 objects misplaced (4.768%); 889 KiB/s, 13 objects/s recovering 2023-10-09T06:19:34.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[77299]: audit 2023-10-09T06:19:33.573224+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[77299]: audit 2023-10-09T06:19:33.574718+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[77299]: cluster 2023-10-09T06:19:33.617661+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 9469/293856 objects degraded (3.222%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[80002]: cluster 2023-10-09T06:19:32.688932+0000 mgr.y (mgr.25723) 428 : cluster 0 pgmap v378: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s; 9469/293856 objects degraded (3.222%); 14012/293856 objects misplaced (4.768%); 889 KiB/s, 13 objects/s recovering 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[80002]: audit 2023-10-09T06:19:33.573224+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[80002]: audit 2023-10-09T06:19:33.574718+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:34.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:34 smithi125 bash[80002]: cluster 2023-10-09T06:19:33.617661+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 9469/293856 objects degraded (3.222%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:34.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:34.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 100854 100838 73.2722 44.6875 0.0215895 0.013642 2023-10-09T06:19:34.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:34.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:34.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:34 smithi155 bash[59495]: audit 2023-10-09T06:19:33.573224+0000 mon.a (mon.0) 573 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:19:34.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:34 smithi155 bash[59495]: audit 2023-10-09T06:19:33.574718+0000 mon.a (mon.0) 574 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:34.318 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:34 smithi155 bash[59495]: cluster 2023-10-09T06:19:33.617661+0000 mon.a (mon.0) 575 : cluster 3 Health check update: Degraded data redundancy: 9469/293856 objects degraded (3.222%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:34.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:34.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:34.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:34.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:34.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:34.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:34.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:34.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:35.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:35.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:35.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:35.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 101573 101557 72.9465 44.9375 0.0127207 0.0137021 2023-10-09T06:19:35.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:35.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:35.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:35.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:35.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:35.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:35.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:35.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:35.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:35.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:35.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:35.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:35.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:35.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:35.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:36.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:36.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:36.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 102281 102265 72.6204 44.25 0.00531644 0.0137631 2023-10-09T06:19:36.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:36.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:36 smithi125 bash[77299]: cluster 2023-10-09T06:19:34.689668+0000 mgr.y (mgr.25723) 429 : cluster 0 pgmap v379: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 9469/298992 objects degraded (3.167%); 13927/298992 objects misplaced (4.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:19:36.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:36 smithi125 bash[80002]: cluster 2023-10-09T06:19:34.689668+0000 mgr.y (mgr.25723) 429 : cluster 0 pgmap v379: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 9469/298992 objects degraded (3.167%); 13927/298992 objects misplaced (4.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:19:36.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:36.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:36.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:36.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:36.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:36 smithi155 bash[59495]: cluster 2023-10-09T06:19:34.689668+0000 mgr.y (mgr.25723) 429 : cluster 0 pgmap v379: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 9469/298992 objects degraded (3.167%); 13927/298992 objects misplaced (4.658%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:19:36.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:36.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:36.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:36.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:36.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:36.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:36.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:36.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:37.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:37.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 102969 102953 72.2875 43 0.0117697 0.0138271 2023-10-09T06:19:37.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:37.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:37 smithi125 bash[77299]: cluster 2023-10-09T06:19:36.690438+0000 mgr.y (mgr.25723) 430 : cluster 0 pgmap v380: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9469/304365 objects degraded (3.111%); 13927/304365 objects misplaced (4.576%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:37.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:37 smithi125 bash[80002]: cluster 2023-10-09T06:19:36.690438+0000 mgr.y (mgr.25723) 430 : cluster 0 pgmap v380: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9469/304365 objects degraded (3.111%); 13927/304365 objects misplaced (4.576%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:37.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:37.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:37 smithi155 bash[59495]: cluster 2023-10-09T06:19:36.690438+0000 mgr.y (mgr.25723) 430 : cluster 0 pgmap v380: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 9469/304365 objects degraded (3.111%); 13927/304365 objects misplaced (4.576%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:37.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:37.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:37.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:37.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:37.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:37.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:37.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:37.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:37.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:38.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:38.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 4 103659 103655 71.9718 43.875 0.0189506 0.013888 2023-10-09T06:19:38.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:38.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:38.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:38.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:38.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:38.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:38.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:38.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:38.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:38 smithi155 bash[59495]: cluster 2023-10-09T06:19:38.618977+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 9469/304365 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:38.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:38.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:38.987 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:38 smithi125 bash[77299]: cluster 2023-10-09T06:19:38.618977+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 9469/304365 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:38.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:38 smithi125 bash[80002]: cluster 2023-10-09T06:19:38.618977+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 9469/304365 objects degraded (3.111%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:39.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:39.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:39.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:39.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0211 2023-10-09T06:19:39.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 103659 2023-10-09T06:19:39.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:19:39.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 71.9685 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 34.4097 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 182 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1151 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 550.566 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2912 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0138888 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0317649 2023-10-09T06:19:39.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 2.36825 2023-10-09T06:19:39.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00139442 2023-10-09T06:19:39.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:39.197 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:19:39.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:39.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:39.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:39.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:39.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:39.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:39.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:39.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:39.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:39.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:39.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:39.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:39 smithi155 bash[59495]: cluster 2023-10-09T06:19:38.691050+0000 mgr.y (mgr.25723) 431 : cluster 0 pgmap v381: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 9469/306387 objects degraded (3.091%); 13927/306387 objects misplaced (4.546%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:39.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:40.002 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:39 smithi125 bash[77299]: cluster 2023-10-09T06:19:38.691050+0000 mgr.y (mgr.25723) 431 : cluster 0 pgmap v381: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 9469/306387 objects degraded (3.091%); 13927/306387 objects misplaced (4.546%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:40.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:39 smithi125 bash[80002]: cluster 2023-10-09T06:19:38.691050+0000 mgr.y (mgr.25723) 431 : cluster 0 pgmap v381: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 9469/306387 objects degraded (3.091%); 13927/306387 objects misplaced (4.546%); 874 KiB/s, 13 objects/s recovering 2023-10-09T06:19:40.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:40.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:40.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:40.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:40.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:40.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:40.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:40.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:40.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:40.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:40.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:40.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:40.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:40.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:40.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:40.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:40.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:41.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:41.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:41.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:41.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:41.241 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:41] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.33.4" 2023-10-09T06:19:41.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:41.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:41.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:41.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:41.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:41.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:41.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:41.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:41.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:41.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:41 smithi155 bash[59495]: cluster 2023-10-09T06:19:40.691711+0000 mgr.y (mgr.25723) 432 : cluster 0 pgmap v382: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 9469/308520 objects degraded (3.069%); 13832/308520 objects misplaced (4.483%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:19:41.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:41 smithi125 bash[77299]: cluster 2023-10-09T06:19:40.691711+0000 mgr.y (mgr.25723) 432 : cluster 0 pgmap v382: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 9469/308520 objects degraded (3.069%); 13832/308520 objects misplaced (4.483%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:19:41.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:41 smithi125 bash[80002]: cluster 2023-10-09T06:19:40.691711+0000 mgr.y (mgr.25723) 432 : cluster 0 pgmap v382: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 9469/308520 objects degraded (3.069%); 13832/308520 objects misplaced (4.483%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:19:41.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:41.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:41.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:42.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:42.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:42.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:42.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:42.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:42.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:42.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:42.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:42.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:42.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:43.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:43.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:43.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:43.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:43.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:43.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:43.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:43.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:43.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:43.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:43.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:43.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:43.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:43 smithi125 bash[77299]: cluster 2023-10-09T06:19:42.692361+0000 mgr.y (mgr.25723) 433 : cluster 0 pgmap v383: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 683 op/s; 9469/308565 objects degraded (3.069%); 13832/308565 objects misplaced (4.483%); 960 KiB/s, 14 objects/s recovering 2023-10-09T06:19:43.873 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:43 smithi125 bash[77299]: cluster 2023-10-09T06:19:43.620164+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 9469/308565 objects degraded (3.069%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:43.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:43 smithi125 bash[80002]: cluster 2023-10-09T06:19:42.692361+0000 mgr.y (mgr.25723) 433 : cluster 0 pgmap v383: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 683 op/s; 9469/308565 objects degraded (3.069%); 13832/308565 objects misplaced (4.483%); 960 KiB/s, 14 objects/s recovering 2023-10-09T06:19:43.873 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:43 smithi125 bash[80002]: cluster 2023-10-09T06:19:43.620164+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 9469/308565 objects degraded (3.069%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:43.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:43.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:43.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:44.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:44.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:43 smithi155 bash[59495]: cluster 2023-10-09T06:19:42.692361+0000 mgr.y (mgr.25723) 433 : cluster 0 pgmap v383: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 683 op/s; 9469/308565 objects degraded (3.069%); 13832/308565 objects misplaced (4.483%); 960 KiB/s, 14 objects/s recovering 2023-10-09T06:19:44.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:43 smithi155 bash[59495]: cluster 2023-10-09T06:19:43.620164+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 9469/308565 objects degraded (3.069%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:44.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:44.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:44.192 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:19:44.207 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:44.207 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:44.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:44.240 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:19:44.299 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:19:44.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:44.338 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:19:44.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:44.385 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:19:44.431 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:19:44.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:44.470 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:19:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:44.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:44.514 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.522 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:44.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:44.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:44.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:44.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:44.780 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:19:44.782 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 5 pgp_num to 16 2023-10-09T06:19:44.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:44.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:44.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:45.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:45.023 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:44 smithi155 bash[59495]: audit 2023-10-09T06:19:43.895615+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:19:45.023 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:44 smithi155 bash[59495]: audit 2023-10-09T06:19:44.133560+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.125:0/3902415349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:45.023 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:44 smithi155 bash[59495]: audit 2023-10-09T06:19:44.659449+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.125:0/2981084482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.023 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:44 smithi155 bash[59495]: audit 2023-10-09T06:19:44.659957+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:45.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[77299]: audit 2023-10-09T06:19:43.895615+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[77299]: audit 2023-10-09T06:19:44.133560+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.125:0/3902415349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[77299]: audit 2023-10-09T06:19:44.659449+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.125:0/2981084482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[77299]: audit 2023-10-09T06:19:44.659957+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[80002]: audit 2023-10-09T06:19:43.895615+0000 mon.a (mon.0) 578 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[80002]: audit 2023-10-09T06:19:44.133560+0000 mon.a (mon.0) 579 : audit 0 from='client.? 172.21.15.125:0/3902415349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[80002]: audit 2023-10-09T06:19:44.659449+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.125:0/2981084482' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:44 smithi125 bash[80002]: audit 2023-10-09T06:19:44.659957+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:45.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:45.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:45.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:45.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:45.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:45.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:45.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:45.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:45.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:45.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:45.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:45.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:45.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: cluster 2023-10-09T06:19:44.693031+0000 mgr.y (mgr.25723) 434 : cluster 0 pgmap v384: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 681 op/s; 9469/303717 objects degraded (3.118%); 13745/303717 objects misplaced (4.526%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: cluster 2023-10-09T06:19:44.751787+0000 mon.a (mon.0) 581 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: audit 2023-10-09T06:19:44.764171+0000 mon.a (mon.0) 582 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: audit 2023-10-09T06:19:44.764248+0000 mon.a (mon.0) 583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: cluster 2023-10-09T06:19:44.764359+0000 mon.a (mon.0) 584 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T06:19:45.989 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[77299]: cluster 2023-10-09T06:19:44.765273+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T06:19:45.990 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: cluster 2023-10-09T06:19:44.693031+0000 mgr.y (mgr.25723) 434 : cluster 0 pgmap v384: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 681 op/s; 9469/303717 objects degraded (3.118%); 13745/303717 objects misplaced (4.526%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:19:45.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: cluster 2023-10-09T06:19:44.751787+0000 mon.a (mon.0) 581 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:45.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: audit 2023-10-09T06:19:44.764171+0000 mon.a (mon.0) 582 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:19:45.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: audit 2023-10-09T06:19:44.764248+0000 mon.a (mon.0) 583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:45.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: cluster 2023-10-09T06:19:44.764359+0000 mon.a (mon.0) 584 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T06:19:45.991 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:45 smithi125 bash[80002]: cluster 2023-10-09T06:19:44.765273+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T06:19:45.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:45.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:45.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:46.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:46.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:46.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: cluster 2023-10-09T06:19:44.693031+0000 mgr.y (mgr.25723) 434 : cluster 0 pgmap v384: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 681 op/s; 9469/303717 objects degraded (3.118%); 13745/303717 objects misplaced (4.526%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:19:46.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: cluster 2023-10-09T06:19:44.751787+0000 mon.a (mon.0) 581 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: audit 2023-10-09T06:19:44.764171+0000 mon.a (mon.0) 582 : audit 1 from='client.? 172.21.15.125:0/2652920471' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:19:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: audit 2023-10-09T06:19:44.764248+0000 mon.a (mon.0) 583 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: cluster 2023-10-09T06:19:44.764359+0000 mon.a (mon.0) 584 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2023-10-09T06:19:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:45 smithi155 bash[59495]: cluster 2023-10-09T06:19:44.765273+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e123 with expected crc 2023-10-09T06:19:46.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:46.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:46.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:46.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:46.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:46.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:46.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:46.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:46.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:46.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:46.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:19:46.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:46.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:46.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:46.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:47.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:47.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:47.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:47.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:47.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:47.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:47.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:47.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:47.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:47.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:47.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:47.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:47.932 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:47 smithi125 bash[77299]: cluster 2023-10-09T06:19:46.693550+0000 mgr.y (mgr.25723) 435 : cluster 0 pgmap v386: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 651 op/s; 9469/298209 objects degraded (3.175%); 13745/298209 objects misplaced (4.609%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:47.932 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:47 smithi125 bash[80002]: cluster 2023-10-09T06:19:46.693550+0000 mgr.y (mgr.25723) 435 : cluster 0 pgmap v386: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 651 op/s; 9469/298209 objects degraded (3.175%); 13745/298209 objects misplaced (4.609%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:47.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:48.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:48.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:47 smithi155 bash[59495]: cluster 2023-10-09T06:19:46.693550+0000 mgr.y (mgr.25723) 435 : cluster 0 pgmap v386: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 651 op/s; 9469/298209 objects degraded (3.175%); 13745/298209 objects misplaced (4.609%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:48.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:48.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:48.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:48.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:48.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:48.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:48.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:48.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:48.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:48.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:48.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:48.773 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:48 smithi155 bash[59495]: audit 2023-10-09T06:19:48.566557+0000 mon.a (mon.0) 585 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:48.773 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:48 smithi155 bash[59495]: cluster 2023-10-09T06:19:48.621349+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 9469/298209 objects degraded (3.175%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:48.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:48.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:48.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:48.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:49.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:48 smithi125 bash[77299]: audit 2023-10-09T06:19:48.566557+0000 mon.a (mon.0) 585 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:49.003 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:48 smithi125 bash[77299]: cluster 2023-10-09T06:19:48.621349+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 9469/298209 objects degraded (3.175%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:49.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:48 smithi125 bash[80002]: audit 2023-10-09T06:19:48.566557+0000 mon.a (mon.0) 585 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:49.004 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:48 smithi125 bash[80002]: cluster 2023-10-09T06:19:48.621349+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 9469/298209 objects degraded (3.175%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:49.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:49.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:49.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:49.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:49.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:49.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:49.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:49.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:49.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:49.773 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:49 smithi125 bash[77299]: cluster 2023-10-09T06:19:48.694166+0000 mgr.y (mgr.25723) 436 : cluster 0 pgmap v387: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 650 op/s; 9469/296211 objects degraded (3.197%); 13745/296211 objects misplaced (4.640%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:49.774 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:49 smithi155 bash[59495]: cluster 2023-10-09T06:19:48.694166+0000 mgr.y (mgr.25723) 436 : cluster 0 pgmap v387: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 650 op/s; 9469/296211 objects degraded (3.197%); 13745/296211 objects misplaced (4.640%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:49.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:49.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:49.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:49.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:49.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:50.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:50.053 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:49 smithi125 bash[80002]: cluster 2023-10-09T06:19:48.694166+0000 mgr.y (mgr.25723) 436 : cluster 0 pgmap v387: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 650 op/s; 9469/296211 objects degraded (3.197%); 13745/296211 objects misplaced (4.640%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:19:50.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:50.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:50.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:50.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:50.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:50.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:50.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:19:50.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:50.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:50.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:50.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:50.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:50.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:50.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:50.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:50.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:51.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:51.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:51.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:51.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:51.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:51.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:51.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:51.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:51.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:51.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:51.602 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:19:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:19:51] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T06:19:51.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:51.643 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:51.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:51.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:51.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:51.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:51.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:51 smithi125 bash[77299]: cluster 2023-10-09T06:19:50.694918+0000 mgr.y (mgr.25723) 437 : cluster 0 pgmap v388: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 671 op/s; 9469/292935 objects degraded (3.232%); 13656/292935 objects misplaced (4.662%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:51.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:51 smithi125 bash[80002]: cluster 2023-10-09T06:19:50.694918+0000 mgr.y (mgr.25723) 437 : cluster 0 pgmap v388: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 671 op/s; 9469/292935 objects degraded (3.232%); 13656/292935 objects misplaced (4.662%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:51.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:51.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:52.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:52.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:52.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:51 smithi155 bash[59495]: cluster 2023-10-09T06:19:50.694918+0000 mgr.y (mgr.25723) 437 : cluster 0 pgmap v388: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 671 op/s; 9469/292935 objects degraded (3.232%); 13656/292935 objects misplaced (4.662%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:52.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:52.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:52.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:52.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:52.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:52.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:52.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:52.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:52.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:52.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:52.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:52.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:53.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:53.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:53.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:53.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.233 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:53.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.345 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:19:53.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:19:53.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:53.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:53.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:53.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:53.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:53.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:53.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:53.781 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:53 smithi155 bash[59495]: cluster 2023-10-09T06:19:52.695571+0000 mgr.y (mgr.25723) 438 : cluster 0 pgmap v389: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 307 KiB/s wr, 702 op/s; 9469/287760 objects degraded (3.291%); 13656/287760 objects misplaced (4.746%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:53.781 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:53 smithi155 bash[59495]: cluster 2023-10-09T06:19:53.622526+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 9469/287760 objects degraded (3.291%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:53.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:53.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:53.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:54.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:54.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:53 smithi125 bash[80002]: cluster 2023-10-09T06:19:52.695571+0000 mgr.y (mgr.25723) 438 : cluster 0 pgmap v389: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 307 KiB/s wr, 702 op/s; 9469/287760 objects degraded (3.291%); 13656/287760 objects misplaced (4.746%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:54.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:53 smithi125 bash[80002]: cluster 2023-10-09T06:19:53.622526+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 9469/287760 objects degraded (3.291%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:54.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:53 smithi125 bash[77299]: cluster 2023-10-09T06:19:52.695571+0000 mgr.y (mgr.25723) 438 : cluster 0 pgmap v389: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 307 KiB/s wr, 702 op/s; 9469/287760 objects degraded (3.291%); 13656/287760 objects misplaced (4.746%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:19:54.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:53 smithi125 bash[77299]: cluster 2023-10-09T06:19:53.622526+0000 mon.a (mon.0) 587 : cluster 3 Health check update: Degraded data redundancy: 9469/287760 objects degraded (3.291%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:54.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:54.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:54.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:54.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:54.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:54.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:54.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:54.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:54.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:54.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:54.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:54.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:54.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:54.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:54.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:55.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:55.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:55.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:55.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:55.493 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:55.493 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:55.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:55.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:55.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:55.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:55.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.817 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:55 smithi125 bash[77299]: cluster 2023-10-09T06:19:54.697583+0000 mgr.y (mgr.25723) 439 : cluster 0 pgmap v390: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9469/282537 objects degraded (3.351%); 13580/282537 objects misplaced (4.806%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:55.818 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:55 smithi125 bash[80002]: cluster 2023-10-09T06:19:54.697583+0000 mgr.y (mgr.25723) 439 : cluster 0 pgmap v390: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9469/282537 objects degraded (3.351%); 13580/282537 objects misplaced (4.806%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:55.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:55.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:55.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:55.950 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:55 smithi155 bash[59495]: cluster 2023-10-09T06:19:54.697583+0000 mgr.y (mgr.25723) 439 : cluster 0 pgmap v390: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 9469/282537 objects degraded (3.351%); 13580/282537 objects misplaced (4.806%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:19:55.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:55.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:56.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:56.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.241 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:56.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:56.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:56.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:56.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:56.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:56.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:56.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:56.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:56.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:56.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:56.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:57.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:57.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:57.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:57.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:57.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:57.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:57.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:57.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:57.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:57.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:57.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:57.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:57.748 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:57.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:57.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:57.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:57.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:57 smithi155 bash[59495]: cluster 2023-10-09T06:19:56.699036+0000 mgr.y (mgr.25723) 440 : cluster 0 pgmap v391: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 9469/277251 objects degraded (3.415%); 13580/277251 objects misplaced (4.898%); 885 KiB/s, 13 objects/s recovering 2023-10-09T06:19:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:57.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:57.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:58.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:58.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:57 smithi125 bash[77299]: cluster 2023-10-09T06:19:56.699036+0000 mgr.y (mgr.25723) 440 : cluster 0 pgmap v391: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 9469/277251 objects degraded (3.415%); 13580/277251 objects misplaced (4.898%); 885 KiB/s, 13 objects/s recovering 2023-10-09T06:19:58.029 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:57 smithi125 bash[80002]: cluster 2023-10-09T06:19:56.699036+0000 mgr.y (mgr.25723) 440 : cluster 0 pgmap v391: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 9469/277251 objects degraded (3.415%); 13580/277251 objects misplaced (4.898%); 885 KiB/s, 13 objects/s recovering 2023-10-09T06:19:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:58.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:58.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:58.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:58.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:58.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:58.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:58.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:58.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:58.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:58.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:58.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:58.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:59.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:59.099 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:58 smithi125 bash[77299]: cluster 2023-10-09T06:19:58.623466+0000 mon.a (mon.0) 588 : cluster 3 Health check update: Degraded data redundancy: 9469/277251 objects degraded (3.415%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:59.101 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:58 smithi125 bash[80002]: cluster 2023-10-09T06:19:58.623466+0000 mon.a (mon.0) 588 : cluster 3 Health check update: Degraded data redundancy: 9469/277251 objects degraded (3.415%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:59.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:59.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:58 smithi155 bash[59495]: cluster 2023-10-09T06:19:58.623466+0000 mon.a (mon.0) 588 : cluster 3 Health check update: Degraded data redundancy: 9469/277251 objects degraded (3.415%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:59.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:59.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:59.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:59.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:59.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:59.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:59.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:59.564 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:19:59.565 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:19:59.565 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:19:59.565 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:19:59.565 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:19:59.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:59.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:59.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:59.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:59.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:59.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:59.951 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:19:59 smithi155 bash[59495]: cluster 2023-10-09T06:19:58.699627+0000 mgr.y (mgr.25723) 441 : cluster 0 pgmap v392: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 9469/275457 objects degraded (3.438%); 13580/275457 objects misplaced (4.930%); 880 KiB/s, 13 objects/s recovering 2023-10-09T06:19:59.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:19:59.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:00.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:00.089 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:19:59 smithi125 bash[77299]: cluster 2023-10-09T06:19:58.699627+0000 mgr.y (mgr.25723) 441 : cluster 0 pgmap v392: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 9469/275457 objects degraded (3.438%); 13580/275457 objects misplaced (4.930%); 880 KiB/s, 13 objects/s recovering 2023-10-09T06:20:00.089 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:19:59 smithi125 bash[80002]: cluster 2023-10-09T06:19:58.699627+0000 mgr.y (mgr.25723) 441 : cluster 0 pgmap v392: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 9469/275457 objects degraded (3.438%); 13580/275457 objects misplaced (4.930%); 880 KiB/s, 13 objects/s recovering 2023-10-09T06:20:00.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:00.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:00.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:00.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:00.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:00.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:00.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:00.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:00.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:00.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:00.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:00.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:00.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000114+0000 mon.a (mon.0) 589 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000152+0000 mon.a (mon.0) 590 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000166+0000 mon.a (mon.0) 591 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000181+0000 mon.a (mon.0) 592 : cluster 3 pg 2.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000195+0000 mon.a (mon.0) 593 : cluster 3 pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000209+0000 mon.a (mon.0) 594 : cluster 3 pg 5.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:00.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.000224+0000 mon.a (mon.0) 595 : cluster 3 pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:00.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:00.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:01.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:01.083 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000114+0000 mon.a (mon.0) 589 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:01.083 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000152+0000 mon.a (mon.0) 590 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:20:01.083 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000166+0000 mon.a (mon.0) 591 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:01.083 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000181+0000 mon.a (mon.0) 592 : cluster 3 pg 2.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-10-09T06:20:01.083 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000195+0000 mon.a (mon.0) 593 : cluster 3 pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000209+0000 mon.a (mon.0) 594 : cluster 3 pg 5.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.000224+0000 mon.a (mon.0) 595 : cluster 3 pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000114+0000 mon.a (mon.0) 589 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000152+0000 mon.a (mon.0) 590 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000166+0000 mon.a (mon.0) 591 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 9469/275457 objects degraded (3.438%), 4 pgs degraded 2023-10-09T06:20:01.084 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000181+0000 mon.a (mon.0) 592 : cluster 3 pg 2.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,0] 2023-10-09T06:20:01.085 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000195+0000 mon.a (mon.0) 593 : cluster 3 pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-10-09T06:20:01.085 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000209+0000 mon.a (mon.0) 594 : cluster 3 pg 5.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:01.085 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.000224+0000 mon.a (mon.0) 595 : cluster 3 pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-10-09T06:20:01.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:01.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:01.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:01.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:01.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:01] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.33.4" 2023-10-09T06:20:01.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:01.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:01.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:01.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:01.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:01.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:01.788 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:01.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:01.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:01.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:02.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:02.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:02.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:01 smithi125 bash[77299]: cluster 2023-10-09T06:20:00.700332+0000 mgr.y (mgr.25723) 442 : cluster 0 pgmap v393: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 9469/272289 objects degraded (3.478%); 13486/272289 objects misplaced (4.953%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:20:02.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:01 smithi125 bash[80002]: cluster 2023-10-09T06:20:00.700332+0000 mgr.y (mgr.25723) 442 : cluster 0 pgmap v393: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 9469/272289 objects degraded (3.478%); 13486/272289 objects misplaced (4.953%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:20:02.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:01 smithi155 bash[59495]: cluster 2023-10-09T06:20:00.700332+0000 mgr.y (mgr.25723) 442 : cluster 0 pgmap v393: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 9469/272289 objects degraded (3.478%); 13486/272289 objects misplaced (4.953%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:20:02.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:02.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:02.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:02.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:02.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:02.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:20:02.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:02.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:02.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:02.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:02.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:02.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:02.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:02.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:02.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:02.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:02.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:03.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:03.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:03.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:03.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:03.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:03.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:03.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:03.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:03.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:03.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:03.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:03.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:03.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:03.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[77299]: cluster 2023-10-09T06:20:02.705817+0000 mgr.y (mgr.25723) 443 : cluster 0 pgmap v394: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9469/267408 objects degraded (3.541%); 13486/267408 objects misplaced (5.043%); 906 KiB/s, 14 objects/s recovering 2023-10-09T06:20:03.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[77299]: audit 2023-10-09T06:20:03.566785+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:03.803 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[77299]: cluster 2023-10-09T06:20:03.624550+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 9469/267408 objects degraded (3.541%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:03.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:03.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:03.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:03.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.063 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[80002]: cluster 2023-10-09T06:20:02.705817+0000 mgr.y (mgr.25723) 443 : cluster 0 pgmap v394: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9469/267408 objects degraded (3.541%); 13486/267408 objects misplaced (5.043%); 906 KiB/s, 14 objects/s recovering 2023-10-09T06:20:04.063 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[80002]: audit 2023-10-09T06:20:03.566785+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:04.063 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:03 smithi125 bash[80002]: cluster 2023-10-09T06:20:03.624550+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 9469/267408 objects degraded (3.541%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:04.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:03 smithi155 bash[59495]: cluster 2023-10-09T06:20:02.705817+0000 mgr.y (mgr.25723) 443 : cluster 0 pgmap v394: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 9469/267408 objects degraded (3.541%); 13486/267408 objects misplaced (5.043%); 906 KiB/s, 14 objects/s recovering 2023-10-09T06:20:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:03 smithi155 bash[59495]: audit 2023-10-09T06:20:03.566785+0000 mon.a (mon.0) 596 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:03 smithi155 bash[59495]: cluster 2023-10-09T06:20:03.624550+0000 mon.a (mon.0) 597 : cluster 3 Health check update: Degraded data redundancy: 9469/267408 objects degraded (3.541%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:04.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:04.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:04.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:04.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:04.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:04.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:04.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:04.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:05.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:05.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:05.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:05.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:05.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:05.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:05.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:05.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:05.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:05.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:05.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:05.813 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:05 smithi155 bash[59495]: cluster 2023-10-09T06:20:04.707186+0000 mgr.y (mgr.25723) 444 : cluster 0 pgmap v395: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 9469/262452 objects degraded (3.608%); 13396/262452 objects misplaced (5.104%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T06:20:05.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:05.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:05.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:06.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:06.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:05 smithi125 bash[77299]: cluster 2023-10-09T06:20:04.707186+0000 mgr.y (mgr.25723) 444 : cluster 0 pgmap v395: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 9469/262452 objects degraded (3.608%); 13396/262452 objects misplaced (5.104%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T06:20:06.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:05 smithi125 bash[80002]: cluster 2023-10-09T06:20:04.707186+0000 mgr.y (mgr.25723) 444 : cluster 0 pgmap v395: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 9469/262452 objects degraded (3.608%); 13396/262452 objects misplaced (5.104%); 1.4 MiB/s, 21 objects/s recovering 2023-10-09T06:20:06.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:06.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:06.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:06.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:06.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:06.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:06.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:06.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:06.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:06.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:06.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:06.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:07.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:07.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:07.230 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:07.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:07.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:07.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:07.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:07.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:07.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:07.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:07.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:07.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:07.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:07.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:07 smithi155 bash[59495]: cluster 2023-10-09T06:20:06.708002+0000 mgr.y (mgr.25723) 445 : cluster 0 pgmap v396: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 9469/257520 objects degraded (3.677%); 13396/257520 objects misplaced (5.202%); 980 KiB/s, 15 objects/s recovering 2023-10-09T06:20:07.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:07.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:07.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:08.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:08.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:08.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:08.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:07 smithi125 bash[77299]: cluster 2023-10-09T06:20:06.708002+0000 mgr.y (mgr.25723) 445 : cluster 0 pgmap v396: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 9469/257520 objects degraded (3.677%); 13396/257520 objects misplaced (5.202%); 980 KiB/s, 15 objects/s recovering 2023-10-09T06:20:08.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:07 smithi125 bash[80002]: cluster 2023-10-09T06:20:06.708002+0000 mgr.y (mgr.25723) 445 : cluster 0 pgmap v396: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 9469/257520 objects degraded (3.677%); 13396/257520 objects misplaced (5.202%); 980 KiB/s, 15 objects/s recovering 2023-10-09T06:20:08.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:08.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:08.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:08.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:08.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:08.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:08.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:08.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:08.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:08.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:08.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:08.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:08.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:08.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:08.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:08 smithi155 bash[59495]: cluster 2023-10-09T06:20:08.625636+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 9469/257520 objects degraded (3.677%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:08.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:08.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:08 smithi125 bash[80002]: cluster 2023-10-09T06:20:08.625636+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 9469/257520 objects degraded (3.677%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:08.987 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:08 smithi125 bash[77299]: cluster 2023-10-09T06:20:08.625636+0000 mon.a (mon.0) 598 : cluster 3 Health check update: Degraded data redundancy: 9469/257520 objects degraded (3.677%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:09.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:09.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:09.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:09.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:09.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:09.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:09.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:09.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:09.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:09.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:09.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:09.844 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:20:09.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:09.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[77299]: cluster 2023-10-09T06:20:08.708551+0000 mgr.y (mgr.25723) 446 : cluster 0 pgmap v397: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9469/255582 objects degraded (3.705%); 13396/255582 objects misplaced (5.241%); 981 KiB/s, 15 objects/s recovering 2023-10-09T06:20:09.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[77299]: audit 2023-10-09T06:20:09.298989+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:09.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[77299]: audit 2023-10-09T06:20:09.302934+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.125:0/3283548505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:09.928 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[80002]: cluster 2023-10-09T06:20:08.708551+0000 mgr.y (mgr.25723) 446 : cluster 0 pgmap v397: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9469/255582 objects degraded (3.705%); 13396/255582 objects misplaced (5.241%); 981 KiB/s, 15 objects/s recovering 2023-10-09T06:20:09.929 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[80002]: audit 2023-10-09T06:20:09.298989+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:09.929 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:09 smithi125 bash[80002]: audit 2023-10-09T06:20:09.302934+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.125:0/3283548505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:09.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:09.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:10.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:10.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:10.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:09 smithi155 bash[59495]: cluster 2023-10-09T06:20:08.708551+0000 mgr.y (mgr.25723) 446 : cluster 0 pgmap v397: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9469/255582 objects degraded (3.705%); 13396/255582 objects misplaced (5.241%); 981 KiB/s, 15 objects/s recovering 2023-10-09T06:20:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:09 smithi155 bash[59495]: audit 2023-10-09T06:20:09.298989+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:09 smithi155 bash[59495]: audit 2023-10-09T06:20:09.302934+0000 mon.b (mon.1) 4 : audit 1 from='client.? 172.21.15.125:0/3283548505' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:10.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:10.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:10.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:10.416 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:20:10.463 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:20:10.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:10.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:10.519 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:20:10.553 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:20:10.594 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:20:10.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:10.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:20:10.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:10.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:10.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:20:10.699 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:20:10.707 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.707 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.707 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.707 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.708 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:10.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:10.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:10.847 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:20:10.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.818190+0000 mon.a (mon.0) 600 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: audit 2023-10-09T06:20:09.831813+0000 mon.a (mon.0) 601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.831971+0000 mon.a (mon.0) 602 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.832276+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.854086+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.855381+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: cluster 2023-10-09T06:20:09.860548+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: audit 2023-10-09T06:20:10.363526+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.125:0/1920197171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:10.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:10 smithi155 bash[59495]: audit 2023-10-09T06:20:10.514390+0000 mon.a (mon.0) 604 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:20:10.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:10.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:11.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.818190+0000 mon.a (mon.0) 600 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: audit 2023-10-09T06:20:09.831813+0000 mon.a (mon.0) 601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.831971+0000 mon.a (mon.0) 602 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.832276+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.854086+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.855381+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: cluster 2023-10-09T06:20:09.860548+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: audit 2023-10-09T06:20:10.363526+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.125:0/1920197171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[77299]: audit 2023-10-09T06:20:10.514390+0000 mon.a (mon.0) 604 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.818190+0000 mon.a (mon.0) 600 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: audit 2023-10-09T06:20:09.831813+0000 mon.a (mon.0) 601 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.831971+0000 mon.a (mon.0) 602 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.832276+0000 osd.4 (osd.4) 51 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.854086+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.855381+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: cluster 2023-10-09T06:20:09.860548+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e124 with expected crc 2023-10-09T06:20:11.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: audit 2023-10-09T06:20:10.363526+0000 mon.a (mon.0) 603 : audit 0 from='client.? 172.21.15.125:0/1920197171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:11.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:10 smithi125 bash[80002]: audit 2023-10-09T06:20:10.514390+0000 mon.a (mon.0) 604 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:20:11.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:11.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:11.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:11.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:11.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:11.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:20:11.459 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:11.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:11.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:11.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:11.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:11.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:11.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:11.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:11.875 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.709050+0000 mgr.y (mgr.25723) 447 : cluster 0 pgmap v399: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 9557/252570 objects degraded (3.784%); 13215/252570 objects misplaced (5.232%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:20:11.875 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: audit 2023-10-09T06:20:10.832667+0000 mon.a (mon.0) 605 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:20:11.875 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.832778+0000 mon.a (mon.0) 606 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T06:20:11.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.834480+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:11.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.838501+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:11.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.839521+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:11.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.851724+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:11.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:11 smithi155 bash[59495]: cluster 2023-10-09T06:20:10.872934+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:11.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:11.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:12.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:12.032 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:12.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.709050+0000 mgr.y (mgr.25723) 447 : cluster 0 pgmap v399: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 9557/252570 objects degraded (3.784%); 13215/252570 objects misplaced (5.232%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: audit 2023-10-09T06:20:10.832667+0000 mon.a (mon.0) 605 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.832778+0000 mon.a (mon.0) 606 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.834480+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.838501+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.839521+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.851724+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[77299]: cluster 2023-10-09T06:20:10.872934+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.709050+0000 mgr.y (mgr.25723) 447 : cluster 0 pgmap v399: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 9557/252570 objects degraded (3.784%); 13215/252570 objects misplaced (5.232%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: audit 2023-10-09T06:20:10.832667+0000 mon.a (mon.0) 605 : audit 1 from='client.? 172.21.15.125:0/4294668618' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.832778+0000 mon.a (mon.0) 606 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.834480+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.838501+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.839521+0000 osd.4 (osd.4) 52 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.851724+0000 osd.4 (osd.4) 53 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:11 smithi125 bash[80002]: cluster 2023-10-09T06:20:10.872934+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-10-09T06:20:12.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:12.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:12.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:12.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:12.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:12.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:12.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:12.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:12.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:12.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:12.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:12.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:13.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:13.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:13.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:13.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:13.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:13.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:13.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:13.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:13.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:13.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:13.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:13.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:13.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:13.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:13.872 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:13 smithi155 bash[59495]: cluster 2023-10-09T06:20:12.709565+0000 mgr.y (mgr.25723) 448 : cluster 0 pgmap v401: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9557/247866 objects degraded (3.856%); 13215/247866 objects misplaced (5.332%); 744 KiB/s, 11 objects/s recovering 2023-10-09T06:20:13.872 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:13 smithi155 bash[59495]: cluster 2023-10-09T06:20:13.626725+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 9557/247866 objects degraded (3.856%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:13.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:13.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:14.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:14.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:13 smithi125 bash[77299]: cluster 2023-10-09T06:20:12.709565+0000 mgr.y (mgr.25723) 448 : cluster 0 pgmap v401: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9557/247866 objects degraded (3.856%); 13215/247866 objects misplaced (5.332%); 744 KiB/s, 11 objects/s recovering 2023-10-09T06:20:14.013 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:13 smithi125 bash[77299]: cluster 2023-10-09T06:20:13.626725+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 9557/247866 objects degraded (3.856%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:14.013 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:13 smithi125 bash[80002]: cluster 2023-10-09T06:20:12.709565+0000 mgr.y (mgr.25723) 448 : cluster 0 pgmap v401: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 9557/247866 objects degraded (3.856%); 13215/247866 objects misplaced (5.332%); 744 KiB/s, 11 objects/s recovering 2023-10-09T06:20:14.014 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:13 smithi125 bash[80002]: cluster 2023-10-09T06:20:13.626725+0000 mon.a (mon.0) 607 : cluster 3 Health check update: Degraded data redundancy: 9557/247866 objects degraded (3.856%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:14.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:14.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:14.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:14.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:20:14.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:14.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:14.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:14.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:20:14.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:14.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:14.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:14.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:14.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:14.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:14.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:14.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:14.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:15.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:15.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:15.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:15.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:15.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:15.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:15.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:15.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:15.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:15.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:15.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:15.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:15.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:15.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:15.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:15.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:15.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:15.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:16.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:15 smithi155 bash[59495]: cluster 2023-10-09T06:20:14.711452+0000 mgr.y (mgr.25723) 449 : cluster 0 pgmap v402: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9678/243069 objects degraded (3.982%); 13003/243069 objects misplaced (5.350%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:16.084 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:16.084 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:16.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:16.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:15 smithi125 bash[77299]: cluster 2023-10-09T06:20:14.711452+0000 mgr.y (mgr.25723) 449 : cluster 0 pgmap v402: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9678/243069 objects degraded (3.982%); 13003/243069 objects misplaced (5.350%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:16.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:15 smithi125 bash[80002]: cluster 2023-10-09T06:20:14.711452+0000 mgr.y (mgr.25723) 449 : cluster 0 pgmap v402: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9678/243069 objects degraded (3.982%); 13003/243069 objects misplaced (5.350%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:16.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:16.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:16.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:16.480 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:16.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:16.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:16.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:16.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:16.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:16.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:16.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:17.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:17.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:17.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:20:17.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:17.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:17.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:17.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:17.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:17.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:17.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:17.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:17.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:17.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:17.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:17 smithi155 bash[59495]: cluster 2023-10-09T06:20:16.712151+0000 mgr.y (mgr.25723) 450 : cluster 0 pgmap v403: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9678/238350 objects degraded (4.060%); 13003/238350 objects misplaced (5.455%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:17.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:17.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:17.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:18.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:18.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:17 smithi125 bash[77299]: cluster 2023-10-09T06:20:16.712151+0000 mgr.y (mgr.25723) 450 : cluster 0 pgmap v403: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9678/238350 objects degraded (4.060%); 13003/238350 objects misplaced (5.455%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:18.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:17 smithi125 bash[80002]: cluster 2023-10-09T06:20:16.712151+0000 mgr.y (mgr.25723) 450 : cluster 0 pgmap v403: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9678/238350 objects degraded (4.060%); 13003/238350 objects misplaced (5.455%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:18.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:18.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:18.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:18.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:18.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:18.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:18.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:18.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:18.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:18.864 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:18 smithi155 bash[59495]: audit 2023-10-09T06:20:18.567093+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:18.864 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:18 smithi155 bash[59495]: cluster 2023-10-09T06:20:18.627674+0000 mon.a (mon.0) 609 : cluster 3 Health check update: Degraded data redundancy: 9678/238350 objects degraded (4.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:18.864 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:18 smithi125 bash[77299]: audit 2023-10-09T06:20:18.567093+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:18.864 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:18 smithi125 bash[77299]: cluster 2023-10-09T06:20:18.627674+0000 mon.a (mon.0) 609 : cluster 3 Health check update: Degraded data redundancy: 9678/238350 objects degraded (4.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:18.883 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:18.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:18.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:18.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.013 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:19.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:19.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:18 smithi125 bash[80002]: audit 2023-10-09T06:20:18.567093+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:19.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:18 smithi125 bash[80002]: cluster 2023-10-09T06:20:18.627674+0000 mon.a (mon.0) 609 : cluster 3 Health check update: Degraded data redundancy: 9678/238350 objects degraded (4.060%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:19.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:19.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:19.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:19.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:19.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:19.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:19.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.718 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:19.718 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:19.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:19.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:19.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:19.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:20.001 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:19 smithi125 bash[77299]: cluster 2023-10-09T06:20:18.712727+0000 mgr.y (mgr.25723) 451 : cluster 0 pgmap v404: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9678/236577 objects degraded (4.091%); 13003/236577 objects misplaced (5.496%); 656 KiB/s, 10 objects/s recovering 2023-10-09T06:20:20.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:19 smithi125 bash[80002]: cluster 2023-10-09T06:20:18.712727+0000 mgr.y (mgr.25723) 451 : cluster 0 pgmap v404: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9678/236577 objects degraded (4.091%); 13003/236577 objects misplaced (5.496%); 656 KiB/s, 10 objects/s recovering 2023-10-09T06:20:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:20.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:20.111 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:19 smithi155 bash[59495]: cluster 2023-10-09T06:20:18.712727+0000 mgr.y (mgr.25723) 451 : cluster 0 pgmap v404: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9678/236577 objects degraded (4.091%); 13003/236577 objects misplaced (5.496%); 656 KiB/s, 10 objects/s recovering 2023-10-09T06:20:20.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:20.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:20.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:20.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:20.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:20.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:20.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:20.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:20.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:20.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:20.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:20.738 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:20.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:20.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:20.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:20.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:20.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:21.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:21.196 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:21.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:21.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:21] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:20:21.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:21.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:21.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:21.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:21.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:21.756 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:21.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:21.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:21.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:22.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:22.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:22.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:21 smithi125 bash[80002]: cluster 2023-10-09T06:20:20.713400+0000 mgr.y (mgr.25723) 452 : cluster 0 pgmap v405: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9806/233514 objects degraded (4.199%); 12788/233514 objects misplaced (5.476%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:20:22.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:21 smithi125 bash[77299]: cluster 2023-10-09T06:20:20.713400+0000 mgr.y (mgr.25723) 452 : cluster 0 pgmap v405: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9806/233514 objects degraded (4.199%); 12788/233514 objects misplaced (5.476%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:20:22.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:21 smithi155 bash[59495]: cluster 2023-10-09T06:20:20.713400+0000 mgr.y (mgr.25723) 452 : cluster 0 pgmap v405: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9806/233514 objects degraded (4.199%); 12788/233514 objects misplaced (5.476%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:20:22.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:22.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:22.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:22.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:22.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:22.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:22.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:22.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:22.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:22.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:22.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:22.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:22.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:22.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:22.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:23.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:23.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:23.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:23.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:23.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:23.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:23.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:23.548 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:23.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:23.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:23.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:23.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:23.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:23.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:23.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:24.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:24.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:24.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:24.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:23 smithi125 bash[77299]: cluster 2023-10-09T06:20:22.713935+0000 mgr.y (mgr.25723) 453 : cluster 0 pgmap v406: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 9806/228696 objects degraded (4.288%); 12788/228696 objects misplaced (5.592%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:20:24.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:23 smithi125 bash[77299]: cluster 2023-10-09T06:20:23.628852+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 9806/228696 objects degraded (4.288%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:24.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:23 smithi125 bash[80002]: cluster 2023-10-09T06:20:22.713935+0000 mgr.y (mgr.25723) 453 : cluster 0 pgmap v406: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 9806/228696 objects degraded (4.288%); 12788/228696 objects misplaced (5.592%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:20:24.126 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:23 smithi125 bash[80002]: cluster 2023-10-09T06:20:23.628852+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 9806/228696 objects degraded (4.288%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:23 smithi155 bash[59495]: cluster 2023-10-09T06:20:22.713935+0000 mgr.y (mgr.25723) 453 : cluster 0 pgmap v406: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 9806/228696 objects degraded (4.288%); 12788/228696 objects misplaced (5.592%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:20:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:23 smithi155 bash[59495]: cluster 2023-10-09T06:20:23.628852+0000 mon.a (mon.0) 610 : cluster 3 Health check update: Degraded data redundancy: 9806/228696 objects degraded (4.288%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:24.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:24.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:24.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:24.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:24.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:24.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:24.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:24.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:24.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:24.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:24.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:24.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:24.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:24.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:25.043 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:25.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:25.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:25.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:25.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:25.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:25.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:25.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:25.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:25.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:25.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:25.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:25.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:25.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:25.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:26.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:26.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:26.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:26.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:25 smithi155 bash[59495]: cluster 2023-10-09T06:20:24.714672+0000 mgr.y (mgr.25723) 454 : cluster 0 pgmap v407: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9931/223857 objects degraded (4.436%); 12567/223857 objects misplaced (5.614%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:26.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:26.175 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:25 smithi125 bash[80002]: cluster 2023-10-09T06:20:24.714672+0000 mgr.y (mgr.25723) 454 : cluster 0 pgmap v407: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9931/223857 objects degraded (4.436%); 12567/223857 objects misplaced (5.614%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:26.176 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:25 smithi125 bash[77299]: cluster 2023-10-09T06:20:24.714672+0000 mgr.y (mgr.25723) 454 : cluster 0 pgmap v407: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9931/223857 objects degraded (4.436%); 12567/223857 objects misplaced (5.614%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:26.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:26.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:26.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:26.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:26.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:26.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:26.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:26.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:26.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:26.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:26.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:26.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:26.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:27.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:27.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:27.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:27.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:27.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:27.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:27.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:27.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:27.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:27.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:27.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:27.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:27.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:27.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:27 smithi125 bash[80002]: cluster 2023-10-09T06:20:26.715538+0000 mgr.y (mgr.25723) 455 : cluster 0 pgmap v408: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9931/219231 objects degraded (4.530%); 12567/219231 objects misplaced (5.732%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:27.911 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:27.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:27.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:28.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:28.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:27 smithi125 bash[77299]: cluster 2023-10-09T06:20:26.715538+0000 mgr.y (mgr.25723) 455 : cluster 0 pgmap v408: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9931/219231 objects degraded (4.530%); 12567/219231 objects misplaced (5.732%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:28.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:27 smithi155 bash[59495]: cluster 2023-10-09T06:20:26.715538+0000 mgr.y (mgr.25723) 455 : cluster 0 pgmap v408: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 9931/219231 objects degraded (4.530%); 12567/219231 objects misplaced (5.732%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:28.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:28.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:28.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:28.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:28.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:28.548 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:28.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:28.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:28.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:28.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:28.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:28.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:28.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:29.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:29.091 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:28 smithi125 bash[77299]: cluster 2023-10-09T06:20:28.629793+0000 mon.a (mon.0) 611 : cluster 3 Health check update: Degraded data redundancy: 9931/219231 objects degraded (4.530%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:29.092 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:28 smithi125 bash[80002]: cluster 2023-10-09T06:20:28.629793+0000 mon.a (mon.0) 611 : cluster 3 Health check update: Degraded data redundancy: 9931/219231 objects degraded (4.530%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:29.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:29.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:29.222 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:28 smithi155 bash[59495]: cluster 2023-10-09T06:20:28.629793+0000 mon.a (mon.0) 611 : cluster 3 Health check update: Degraded data redundancy: 9931/219231 objects degraded (4.530%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:29.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:29.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:29.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:29.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:29.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:29.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:29.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:29.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:29.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:29.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:29.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:30.012 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:30.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:30.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:29 smithi155 bash[59495]: cluster 2023-10-09T06:20:28.716180+0000 mgr.y (mgr.25723) 456 : cluster 0 pgmap v409: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9931/217392 objects degraded (4.568%); 12567/217392 objects misplaced (5.781%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:30.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:30.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:30.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:30.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:29 smithi125 bash[77299]: cluster 2023-10-09T06:20:28.716180+0000 mgr.y (mgr.25723) 456 : cluster 0 pgmap v409: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9931/217392 objects degraded (4.568%); 12567/217392 objects misplaced (5.781%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:30.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:29 smithi125 bash[80002]: cluster 2023-10-09T06:20:28.716180+0000 mgr.y (mgr.25723) 456 : cluster 0 pgmap v409: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9931/217392 objects degraded (4.568%); 12567/217392 objects misplaced (5.781%); 976 KiB/s, 15 objects/s recovering 2023-10-09T06:20:30.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:30.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:30.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:30.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:30.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:30.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:30.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:30.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:30.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:30.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:30.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:30.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:31.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:31.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:31.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:31.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:31] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:20:31.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:31.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:31.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:31.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:31.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:31.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:31.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:31.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:31.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:31.917 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:20:31.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:31.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:31.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:32.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:32.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:32.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[77299]: cluster 2023-10-09T06:20:30.716880+0000 mgr.y (mgr.25723) 457 : cluster 0 pgmap v410: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10039/214380 objects degraded (4.683%); 12365/214380 objects misplaced (5.768%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:20:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[77299]: audit 2023-10-09T06:20:31.544182+0000 mon.a (mon.0) 612 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:20:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[77299]: audit 2023-10-09T06:20:31.800541+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:32.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[80002]: cluster 2023-10-09T06:20:30.716880+0000 mgr.y (mgr.25723) 457 : cluster 0 pgmap v410: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10039/214380 objects degraded (4.683%); 12365/214380 objects misplaced (5.768%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:20:32.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[80002]: audit 2023-10-09T06:20:31.544182+0000 mon.a (mon.0) 612 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:20:32.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:31 smithi125 bash[80002]: audit 2023-10-09T06:20:31.800541+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:32.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:32.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:32.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:32.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:31 smithi155 bash[59495]: cluster 2023-10-09T06:20:30.716880+0000 mgr.y (mgr.25723) 457 : cluster 0 pgmap v410: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10039/214380 objects degraded (4.683%); 12365/214380 objects misplaced (5.768%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:20:32.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:31 smithi155 bash[59495]: audit 2023-10-09T06:20:31.544182+0000 mon.a (mon.0) 612 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:20:32.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:31 smithi155 bash[59495]: audit 2023-10-09T06:20:31.800541+0000 mon.a (mon.0) 613 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:32.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:32.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:32.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:32.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:32.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:32.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:32.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:32.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:32.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:32.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:32.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: audit 2023-10-09T06:20:31.901482+0000 mon.a (mon.0) 614 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: cluster 2023-10-09T06:20:31.901556+0000 mon.a (mon.0) 615 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: cluster 2023-10-09T06:20:31.905961+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: cluster 2023-10-09T06:20:31.915794+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: cluster 2023-10-09T06:20:31.925835+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.005 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:32 smithi155 bash[59495]: cluster 2023-10-09T06:20:31.932411+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:33.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: audit 2023-10-09T06:20:31.901482+0000 mon.a (mon.0) 614 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: cluster 2023-10-09T06:20:31.901556+0000 mon.a (mon.0) 615 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: cluster 2023-10-09T06:20:31.905961+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: cluster 2023-10-09T06:20:31.915794+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: cluster 2023-10-09T06:20:31.925835+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[77299]: cluster 2023-10-09T06:20:31.932411+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: audit 2023-10-09T06:20:31.901482+0000 mon.a (mon.0) 614 : audit 1 from='client.? 172.21.15.125:0/115795068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:20:33.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: cluster 2023-10-09T06:20:31.901556+0000 mon.a (mon.0) 615 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-09T06:20:33.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: cluster 2023-10-09T06:20:31.905961+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: cluster 2023-10-09T06:20:31.915794+0000 osd.4 (osd.4) 54 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: cluster 2023-10-09T06:20:31.925835+0000 osd.4 (osd.4) 55 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:32 smithi125 bash[80002]: cluster 2023-10-09T06:20:31.932411+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e126 with expected crc 2023-10-09T06:20:33.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:33.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:33.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:33.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:33.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:33.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:33.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:33.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:33.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:33.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:33.714 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:33.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:33.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:33.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:33.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:33.988 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:20:34.029 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:20:34.066 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:20:34.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:34.106 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:20:34.111 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[77299]: cluster 2023-10-09T06:20:32.717466+0000 mgr.y (mgr.25723) 458 : cluster 0 pgmap v412: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10039/209625 objects degraded (4.789%); 12365/209625 objects misplaced (5.899%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:20:34.111 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[77299]: audit 2023-10-09T06:20:33.567529+0000 mon.a (mon.0) 616 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:34.111 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[77299]: cluster 2023-10-09T06:20:33.630715+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 10039/209625 objects degraded (4.789%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:34.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[80002]: cluster 2023-10-09T06:20:32.717466+0000 mgr.y (mgr.25723) 458 : cluster 0 pgmap v412: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10039/209625 objects degraded (4.789%); 12365/209625 objects misplaced (5.899%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:20:34.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[80002]: audit 2023-10-09T06:20:33.567529+0000 mon.a (mon.0) 616 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:34.112 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:33 smithi125 bash[80002]: cluster 2023-10-09T06:20:33.630715+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 10039/209625 objects degraded (4.789%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:34.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:34.170 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:20:34.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:20:34.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:34.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:33 smithi155 bash[59495]: cluster 2023-10-09T06:20:32.717466+0000 mgr.y (mgr.25723) 458 : cluster 0 pgmap v412: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10039/209625 objects degraded (4.789%); 12365/209625 objects misplaced (5.899%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:20:34.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:33 smithi155 bash[59495]: audit 2023-10-09T06:20:33.567529+0000 mon.a (mon.0) 616 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:34.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:33 smithi155 bash[59495]: cluster 2023-10-09T06:20:33.630715+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 10039/209625 objects degraded (4.789%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:34.232 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:20:34.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:34.299 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:20:34.344 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:20:34.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.358 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.359 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:34.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:140012 5.0 5:018833ab:::benchmark_data_smithi125_106927_object36350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e126)", 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:20:34.440946+0000", 2023-10-09T06:20:34.454 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0054217739999999999, 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054591910000000004, 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 140012 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:20:34.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440946+0000", 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440946+0000", 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440944+0000", 2023-10-09T06:20:34.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440954+0000", 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6379999999999999e-06 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440970+0000", 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5735000000000001e-05 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.440978+0000", 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0730000000000005e-06 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.443075+0000", 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020972230000000001 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.443312+0000", 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023693100000000001 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:20:34.443568+0000", 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025646700000000002 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T06:20:34.459 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 1 2023-10-09T06:20:34.460 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:20:34.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:34.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:34.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:34.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:34.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:34.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:34.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:34.952 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:20:34.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:34.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:35.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:35.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:35.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[77299]: audit 2023-10-09T06:20:33.928020+0000 mon.a (mon.0) 618 : audit 0 from='client.? 172.21.15.125:0/3812167270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:35.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[77299]: audit 2023-10-09T06:20:34.866763+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.125:0/635577200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[77299]: audit 2023-10-09T06:20:34.868152+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[80002]: audit 2023-10-09T06:20:33.928020+0000 mon.a (mon.0) 618 : audit 0 from='client.? 172.21.15.125:0/3812167270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:35.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[80002]: audit 2023-10-09T06:20:34.866763+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.125:0/635577200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:34 smithi125 bash[80002]: audit 2023-10-09T06:20:34.868152+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:35.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:35.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:34 smithi155 bash[59495]: audit 2023-10-09T06:20:33.928020+0000 mon.a (mon.0) 618 : audit 0 from='client.? 172.21.15.125:0/3812167270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:35.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:34 smithi155 bash[59495]: audit 2023-10-09T06:20:34.866763+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.125:0/635577200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:34 smithi155 bash[59495]: audit 2023-10-09T06:20:34.868152+0000 mon.a (mon.0) 619 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:35.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:35.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:35.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:35.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:35.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:36.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:36.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.718156+0000 mgr.y (mgr.25723) 459 : cluster 0 pgmap v413: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10039/206307 objects degraded (4.866%); 12365/206307 objects misplaced (5.993%); 601 KiB/s, 9 objects/s recovering 2023-10-09T06:20:36.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.914532+0000 mon.a (mon.0) 620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: audit 2023-10-09T06:20:34.932948+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.933045+0000 mon.a (mon.0) 622 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.933795+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.942663+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[77299]: cluster 2023-10-09T06:20:34.958222+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.718156+0000 mgr.y (mgr.25723) 459 : cluster 0 pgmap v413: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10039/206307 objects degraded (4.866%); 12365/206307 objects misplaced (5.993%); 601 KiB/s, 9 objects/s recovering 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.914532+0000 mon.a (mon.0) 620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: audit 2023-10-09T06:20:34.932948+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:36.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.933045+0000 mon.a (mon.0) 622 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T06:20:36.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.933795+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.942663+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:35 smithi125 bash[80002]: cluster 2023-10-09T06:20:34.958222+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.718156+0000 mgr.y (mgr.25723) 459 : cluster 0 pgmap v413: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10039/206307 objects degraded (4.866%); 12365/206307 objects misplaced (5.993%); 601 KiB/s, 9 objects/s recovering 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.914532+0000 mon.a (mon.0) 620 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: audit 2023-10-09T06:20:34.932948+0000 mon.a (mon.0) 621 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.933045+0000 mon.a (mon.0) 622 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.933795+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.942663+0000 osd.4 (osd.4) 56 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:35 smithi155 bash[59495]: cluster 2023-10-09T06:20:34.958222+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e127 with expected crc 2023-10-09T06:20:36.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:36.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:36.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:36.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:36.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:36.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:36.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:36.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:36.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:36.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:36.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:36.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:36.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:36.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:36.977 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:37.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:37.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:37.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:37.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:20:37.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:37.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:37.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:37.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:37.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:37.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:37.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:37.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:37.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:37.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:37.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:37.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:37.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:37.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:37.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:38.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:38.098 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:37 smithi155 bash[59495]: cluster 2023-10-09T06:20:36.718847+0000 mgr.y (mgr.25723) 460 : cluster 0 pgmap v415: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 10111/200559 objects degraded (5.041%); 12188/200559 objects misplaced (6.077%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:20:38.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:38.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:38.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:38.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:38.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:38.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:38.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:37 smithi125 bash[77299]: cluster 2023-10-09T06:20:36.718847+0000 mgr.y (mgr.25723) 460 : cluster 0 pgmap v415: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 10111/200559 objects degraded (5.041%); 12188/200559 objects misplaced (6.077%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:20:38.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:37 smithi125 bash[80002]: cluster 2023-10-09T06:20:36.718847+0000 mgr.y (mgr.25723) 460 : cluster 0 pgmap v415: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 10111/200559 objects degraded (5.041%); 12188/200559 objects misplaced (6.077%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:20:38.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:38.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:38.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:38.642 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:38.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:38.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:38.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:38.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:38.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:39.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:39.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:39.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:39.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:39.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:39.177 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:20:39.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:39.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:39.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:39.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:39.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:39.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:39.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:39.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:39 smithi155 bash[59495]: audit 2023-10-09T06:20:38.572176+0000 mon.a (mon.0) 623 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:39 smithi155 bash[59495]: audit 2023-10-09T06:20:38.586773+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:39 smithi155 bash[59495]: cluster 2023-10-09T06:20:38.633170+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10111/200559 objects degraded (5.041%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:39.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:39 smithi155 bash[59495]: cluster 2023-10-09T06:20:38.719512+0000 mgr.y (mgr.25723) 461 : cluster 0 pgmap v416: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10111/198972 objects degraded (5.082%); 12188/198972 objects misplaced (6.125%); 840 KiB/s, 13 objects/s recovering 2023-10-09T06:20:39.606 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:39.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:39.724 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[77299]: audit 2023-10-09T06:20:38.572176+0000 mon.a (mon.0) 623 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[77299]: audit 2023-10-09T06:20:38.586773+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[77299]: cluster 2023-10-09T06:20:38.633170+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10111/200559 objects degraded (5.041%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[77299]: cluster 2023-10-09T06:20:38.719512+0000 mgr.y (mgr.25723) 461 : cluster 0 pgmap v416: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10111/198972 objects degraded (5.082%); 12188/198972 objects misplaced (6.125%); 840 KiB/s, 13 objects/s recovering 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[80002]: audit 2023-10-09T06:20:38.572176+0000 mon.a (mon.0) 623 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[80002]: audit 2023-10-09T06:20:38.586773+0000 mon.a (mon.0) 624 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[80002]: cluster 2023-10-09T06:20:38.633170+0000 mon.a (mon.0) 625 : cluster 3 Health check update: Degraded data redundancy: 10111/200559 objects degraded (5.041%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:39.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:39 smithi125 bash[80002]: cluster 2023-10-09T06:20:38.719512+0000 mgr.y (mgr.25723) 461 : cluster 0 pgmap v416: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10111/198972 objects degraded (5.082%); 12188/198972 objects misplaced (6.125%); 840 KiB/s, 13 objects/s recovering 2023-10-09T06:20:39.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:39.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:39.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:39.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:40.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:40.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:40.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:40.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:40.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:40.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:40.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:40.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:40.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:40.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:40.703 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:40.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:40.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:40.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:40.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:40.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:41.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:41.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:41.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:41.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:41.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:41.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:41.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:41.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:41.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:41.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:41] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:20:41.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.722 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:41.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:41.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:41.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:41.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:41.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:42.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:42.063 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: cluster 2023-10-09T06:20:40.720233+0000 mgr.y (mgr.25723) 462 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 10197/196074 objects degraded (5.201%); 12009/196074 objects misplaced (6.125%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:42.063 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: audit 2023-10-09T06:20:41.054015+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.063 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: audit 2023-10-09T06:20:41.065737+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: audit 2023-10-09T06:20:41.645342+0000 mon.a (mon.0) 628 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: audit 2023-10-09T06:20:41.646440+0000 mon.a (mon.0) 629 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.064 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[77299]: audit 2023-10-09T06:20:41.655192+0000 mon.a (mon.0) 630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:42.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:42.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:42.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: cluster 2023-10-09T06:20:40.720233+0000 mgr.y (mgr.25723) 462 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 10197/196074 objects degraded (5.201%); 12009/196074 objects misplaced (6.125%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: audit 2023-10-09T06:20:41.054015+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: audit 2023-10-09T06:20:41.065737+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: audit 2023-10-09T06:20:41.645342+0000 mon.a (mon.0) 628 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: audit 2023-10-09T06:20:41.646440+0000 mon.a (mon.0) 629 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.317 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:42 smithi125 bash[80002]: audit 2023-10-09T06:20:41.655192+0000 mon.a (mon.0) 630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:42.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: cluster 2023-10-09T06:20:40.720233+0000 mgr.y (mgr.25723) 462 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 10197/196074 objects degraded (5.201%); 12009/196074 objects misplaced (6.125%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:20:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: audit 2023-10-09T06:20:41.054015+0000 mon.a (mon.0) 626 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: audit 2023-10-09T06:20:41.065737+0000 mon.a (mon.0) 627 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: audit 2023-10-09T06:20:41.645342+0000 mon.a (mon.0) 628 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: audit 2023-10-09T06:20:41.646440+0000 mon.a (mon.0) 629 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:42 smithi155 bash[59495]: audit 2023-10-09T06:20:41.655192+0000 mon.a (mon.0) 630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:42.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:42.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:42.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:42.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:42.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:42.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:42.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:42.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:42.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:42.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:42.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:42.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:42.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:43.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:43.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:43.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:43.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:43.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:43.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:43.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:43.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:43.756 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:43.756 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:43.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:43.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:43.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:43.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:44.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:44.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:44.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:44.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:44 smithi125 bash[77299]: cluster 2023-10-09T06:20:42.720875+0000 mgr.y (mgr.25723) 463 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10197/191547 objects degraded (5.323%); 12009/191547 objects misplaced (6.269%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:44.168 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:44 smithi125 bash[77299]: cluster 2023-10-09T06:20:43.634824+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 10197/191547 objects degraded (5.323%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:44.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:44 smithi125 bash[80002]: cluster 2023-10-09T06:20:42.720875+0000 mgr.y (mgr.25723) 463 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10197/191547 objects degraded (5.323%); 12009/191547 objects misplaced (6.269%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:44.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:44 smithi125 bash[80002]: cluster 2023-10-09T06:20:43.634824+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 10197/191547 objects degraded (5.323%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:44.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:44.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:44.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:44.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:44.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:44 smithi155 bash[59495]: cluster 2023-10-09T06:20:42.720875+0000 mgr.y (mgr.25723) 463 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10197/191547 objects degraded (5.323%); 12009/191547 objects misplaced (6.269%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:44.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:44 smithi155 bash[59495]: cluster 2023-10-09T06:20:43.634824+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 10197/191547 objects degraded (5.323%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:44.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:44.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:44.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:44.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:44.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:44.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:44.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:44.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:44.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:44.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:45.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:45.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:45.159 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:45.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:45.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:45.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.434 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:45.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:45.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:45.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:45.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:45.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:45.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:45.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:46.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:46.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:46.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:46.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:46.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:46.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:46.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:46 smithi125 bash[77299]: cluster 2023-10-09T06:20:44.721535+0000 mgr.y (mgr.25723) 464 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 10197/188241 objects degraded (5.417%); 12009/188241 objects misplaced (6.380%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:46.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:46 smithi125 bash[80002]: cluster 2023-10-09T06:20:44.721535+0000 mgr.y (mgr.25723) 464 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 10197/188241 objects degraded (5.417%); 12009/188241 objects misplaced (6.380%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:46.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:46 smithi155 bash[59495]: cluster 2023-10-09T06:20:44.721535+0000 mgr.y (mgr.25723) 464 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 10197/188241 objects degraded (5.417%); 12009/188241 objects misplaced (6.380%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:46.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:46.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:46.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:46.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:46.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:46.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:46.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:46.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:46.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:46.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:46.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:47.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:47.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:47.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:47.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:47.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:47.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:47.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:47.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:47.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:47.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:47.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:47.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:47.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:47.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:47.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:47.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:47.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:48.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:48.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:48.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:48.221 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:48 smithi125 bash[77299]: cluster 2023-10-09T06:20:46.722297+0000 mgr.y (mgr.25723) 465 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 10299/182346 objects degraded (5.648%); 11813/182346 objects misplaced (6.478%); 1015 KiB/s, 15 objects/s recovering 2023-10-09T06:20:48.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:48 smithi125 bash[80002]: cluster 2023-10-09T06:20:46.722297+0000 mgr.y (mgr.25723) 465 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 10299/182346 objects degraded (5.648%); 11813/182346 objects misplaced (6.478%); 1015 KiB/s, 15 objects/s recovering 2023-10-09T06:20:48.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:48.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:48.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:48.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:48 smithi155 bash[59495]: cluster 2023-10-09T06:20:46.722297+0000 mgr.y (mgr.25723) 465 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 10299/182346 objects degraded (5.648%); 11813/182346 objects misplaced (6.478%); 1015 KiB/s, 15 objects/s recovering 2023-10-09T06:20:48.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:48.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:48.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:48.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:48.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:48.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:48.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:48.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:48.939 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:48.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:48.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:49.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:49.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:49.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:49.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:49.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:49.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:49.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:49.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:49.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[77299]: audit 2023-10-09T06:20:48.578056+0000 mon.a (mon.0) 632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:49.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[77299]: audit 2023-10-09T06:20:48.579402+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:49.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[77299]: cluster 2023-10-09T06:20:48.635916+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10299/182346 objects degraded (5.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:49.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[77299]: cluster 2023-10-09T06:20:48.722920+0000 mgr.y (mgr.25723) 466 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 10299/180501 objects degraded (5.706%); 11813/180501 objects misplaced (6.545%); 997 KiB/s, 15 objects/s recovering 2023-10-09T06:20:49.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[80002]: audit 2023-10-09T06:20:48.578056+0000 mon.a (mon.0) 632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:49.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[80002]: audit 2023-10-09T06:20:48.579402+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:49.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[80002]: cluster 2023-10-09T06:20:48.635916+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10299/182346 objects degraded (5.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:49.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:49 smithi125 bash[80002]: cluster 2023-10-09T06:20:48.722920+0000 mgr.y (mgr.25723) 466 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 10299/180501 objects degraded (5.706%); 11813/180501 objects misplaced (6.545%); 997 KiB/s, 15 objects/s recovering 2023-10-09T06:20:49.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:49.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:49.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:49.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:49.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:49 smithi155 bash[59495]: audit 2023-10-09T06:20:48.578056+0000 mon.a (mon.0) 632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:20:49.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:49 smithi155 bash[59495]: audit 2023-10-09T06:20:48.579402+0000 mon.a (mon.0) 633 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:49.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:49 smithi155 bash[59495]: cluster 2023-10-09T06:20:48.635916+0000 mon.a (mon.0) 634 : cluster 3 Health check update: Degraded data redundancy: 10299/182346 objects degraded (5.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:49.790 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:49 smithi155 bash[59495]: cluster 2023-10-09T06:20:48.722920+0000 mgr.y (mgr.25723) 466 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 10299/180501 objects degraded (5.706%); 11813/180501 objects misplaced (6.545%); 997 KiB/s, 15 objects/s recovering 2023-10-09T06:20:49.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:49.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:49.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:49.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:50.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:50.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:50.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:50.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:50.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:50.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:50.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:50.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:50.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:50.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:50.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:50.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:50.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:50.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:50.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:51.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:51.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:51.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:51.169 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:51.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:51.282 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:20:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:20:51] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-10-09T06:20:51.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:51.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:51.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:51.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:51.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:51.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:51.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:51.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:51.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:51.784 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:51 smithi155 bash[59495]: cluster 2023-10-09T06:20:50.723624+0000 mgr.y (mgr.25723) 467 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10374/177558 objects degraded (5.843%); 11636/177558 objects misplaced (6.553%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:20:51.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:51.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:51.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:51.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:52.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:52.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:52.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:52.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:52.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:51 smithi125 bash[77299]: cluster 2023-10-09T06:20:50.723624+0000 mgr.y (mgr.25723) 467 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10374/177558 objects degraded (5.843%); 11636/177558 objects misplaced (6.553%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:20:52.157 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:51 smithi125 bash[80002]: cluster 2023-10-09T06:20:50.723624+0000 mgr.y (mgr.25723) 467 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10374/177558 objects degraded (5.843%); 11636/177558 objects misplaced (6.553%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:20:52.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:52.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:52.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:52.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:52.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:52.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:52.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:52.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:52.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:52.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:52.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:52.812 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:20:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:52.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:52.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:52.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:53.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:52 smithi125 bash[77299]: audit 2023-10-09T06:20:52.769197+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.125:0/3167624259' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:52 smithi125 bash[77299]: audit 2023-10-09T06:20:52.770091+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.030 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:52 smithi125 bash[80002]: audit 2023-10-09T06:20:52.769197+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.125:0/3167624259' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.030 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:52 smithi125 bash[80002]: audit 2023-10-09T06:20:52.770091+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:53.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:52 smithi155 bash[59495]: audit 2023-10-09T06:20:52.769197+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.125:0/3167624259' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:52 smithi155 bash[59495]: audit 2023-10-09T06:20:52.770091+0000 mon.a (mon.0) 635 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:53.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:53.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:53.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:53.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:53.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:53.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.600 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:20:53.600 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:20:53.600 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:20:53.600 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:20:53.600 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:20:53.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:53.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:53.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:53.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:53.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: cluster 2023-10-09T06:20:52.725356+0000 mgr.y (mgr.25723) 468 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 10374/172926 objects degraded (5.999%); 11636/172926 objects misplaced (6.729%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: cluster 2023-10-09T06:20:52.797363+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: cluster 2023-10-09T06:20:52.797515+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: audit 2023-10-09T06:20:52.798868+0000 mon.a (mon.0) 636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: cluster 2023-10-09T06:20:52.799045+0000 mon.a (mon.0) 637 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:20:53.987 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[80002]: cluster 2023-10-09T06:20:53.636983+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 10374/172926 objects degraded (5.999%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: cluster 2023-10-09T06:20:52.725356+0000 mgr.y (mgr.25723) 468 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 10374/172926 objects degraded (5.999%); 11636/172926 objects misplaced (6.729%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: cluster 2023-10-09T06:20:52.797363+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: cluster 2023-10-09T06:20:52.797515+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: audit 2023-10-09T06:20:52.798868+0000 mon.a (mon.0) 636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: cluster 2023-10-09T06:20:52.799045+0000 mon.a (mon.0) 637 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:20:53.988 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:53 smithi125 bash[77299]: cluster 2023-10-09T06:20:53.636983+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 10374/172926 objects degraded (5.999%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:54.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:54.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:54.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: cluster 2023-10-09T06:20:52.725356+0000 mgr.y (mgr.25723) 468 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 10374/172926 objects degraded (5.999%); 11636/172926 objects misplaced (6.729%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:20:54.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: cluster 2023-10-09T06:20:52.797363+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:54.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: cluster 2023-10-09T06:20:52.797515+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e128 with expected crc 2023-10-09T06:20:54.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: audit 2023-10-09T06:20:52.798868+0000 mon.a (mon.0) 636 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:54.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: cluster 2023-10-09T06:20:52.799045+0000 mon.a (mon.0) 637 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:20:54.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:53 smithi155 bash[59495]: cluster 2023-10-09T06:20:53.636983+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 10374/172926 objects degraded (5.999%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:54.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:54.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:54.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:54.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:54.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:54.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:54.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:54.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:54.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:54.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:54.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:54.939 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:54.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:54.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:55.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:55.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:55.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:55.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:55.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:55.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:55.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:55.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:55.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:55.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:55.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:55.742 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:55.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:55.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:55 smithi155 bash[59495]: cluster 2023-10-09T06:20:54.730266+0000 mgr.y (mgr.25723) 469 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 10374/169608 objects degraded (6.116%); 11636/169608 objects misplaced (6.861%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:55.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:55.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:56.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:56.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:56.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:55 smithi125 bash[77299]: cluster 2023-10-09T06:20:54.730266+0000 mgr.y (mgr.25723) 469 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 10374/169608 objects degraded (6.116%); 11636/169608 objects misplaced (6.861%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:56.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:55 smithi125 bash[80002]: cluster 2023-10-09T06:20:54.730266+0000 mgr.y (mgr.25723) 469 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 10374/169608 objects degraded (6.116%); 11636/169608 objects misplaced (6.861%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:20:56.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:56.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:56.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:56.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:56.421 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:56.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:56.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:56.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:56.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:56.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:56.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:56.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:20:56.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:56.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:57.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:57.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:57.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:57.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:57.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:20:57.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:57.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:57.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:57.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:57.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:57.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:57.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:57.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:57.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:57.889 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:57 smithi155 bash[59495]: cluster 2023-10-09T06:20:56.730988+0000 mgr.y (mgr.25723) 470 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10473/163821 objects degraded (6.393%); 11437/163821 objects misplaced (6.981%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:20:57.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:57.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:57.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:58.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:58.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:57 smithi125 bash[77299]: cluster 2023-10-09T06:20:56.730988+0000 mgr.y (mgr.25723) 470 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10473/163821 objects degraded (6.393%); 11437/163821 objects misplaced (6.981%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:20:58.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:57 smithi125 bash[80002]: cluster 2023-10-09T06:20:56.730988+0000 mgr.y (mgr.25723) 470 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10473/163821 objects degraded (6.393%); 11437/163821 objects misplaced (6.981%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:20:58.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:58.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:58.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:58.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:58.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:58.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:58.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:58.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:58.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:58.814 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:58 smithi125 bash[77299]: cluster 2023-10-09T06:20:58.637859+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 10473/163821 objects degraded (6.393%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:58.815 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:58 smithi125 bash[80002]: cluster 2023-10-09T06:20:58.637859+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 10473/163821 objects degraded (6.393%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:58.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:58.933 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:20:58.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:59.145 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:20:59.145 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:20:59.145 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:20:59.146 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:20:59.146 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:20:59.146 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:20:59.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:59.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:58 smithi155 bash[59495]: cluster 2023-10-09T06:20:58.637859+0000 mon.a (mon.0) 639 : cluster 3 Health check update: Degraded data redundancy: 10473/163821 objects degraded (6.393%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:59.179 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:20:59.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:59.314 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.314 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.315 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:59.348 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:59.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:59.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:59.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:59.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:59.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:59.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:59.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:59.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:59.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:20:59.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:59.913 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:20:59.913 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[77299]: cluster 2023-10-09T06:20:58.731696+0000 mgr.y (mgr.25723) 471 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 10473/161973 objects degraded (6.466%); 11437/161973 objects misplaced (7.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:00.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[77299]: audit 2023-10-09T06:20:58.855337+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.125:0/1222805006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:00.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[77299]: audit 2023-10-09T06:20:59.074018+0000 mon.a (mon.0) 641 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:00.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[80002]: cluster 2023-10-09T06:20:58.731696+0000 mgr.y (mgr.25723) 471 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 10473/161973 objects degraded (6.466%); 11437/161973 objects misplaced (7.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:00.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[80002]: audit 2023-10-09T06:20:58.855337+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.125:0/1222805006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:00.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:20:59 smithi125 bash[80002]: audit 2023-10-09T06:20:59.074018+0000 mon.a (mon.0) 641 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:00.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:00.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:59 smithi155 bash[59495]: cluster 2023-10-09T06:20:58.731696+0000 mgr.y (mgr.25723) 471 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 10473/161973 objects degraded (6.466%); 11437/161973 objects misplaced (7.061%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:00.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:59 smithi155 bash[59495]: audit 2023-10-09T06:20:58.855337+0000 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.125:0/1222805006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:00.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:20:59 smithi155 bash[59495]: audit 2023-10-09T06:20:59.074018+0000 mon.a (mon.0) 641 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:00.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:00.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:00.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:00.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:00.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:00.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:00.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:00.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:00.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:00.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:00.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:00.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.808474+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: audit 2023-10-09T06:20:59.817186+0000 mon.a (mon.0) 643 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.817302+0000 mon.a (mon.0) 644 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.828740+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.830678+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.835521+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.842786+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.949 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[77299]: cluster 2023-10-09T06:20:59.846081+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.808474+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: audit 2023-10-09T06:20:59.817186+0000 mon.a (mon.0) 643 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.817302+0000 mon.a (mon.0) 644 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.828740+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.830678+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.835521+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.842786+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.950 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:00 smithi125 bash[80002]: cluster 2023-10-09T06:20:59.846081+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:00.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:00.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:00.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:01.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:01.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.808474+0000 mon.a (mon.0) 642 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:01.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: audit 2023-10-09T06:20:59.817186+0000 mon.a (mon.0) 643 : audit 1 from='client.? 172.21.15.125:0/89911001' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:01.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.817302+0000 mon.a (mon.0) 644 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:21:01.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.828740+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:01.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.830678+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:01.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.835521+0000 osd.4 (osd.4) 57 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:01.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.842786+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:01.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:00 smithi155 bash[59495]: cluster 2023-10-09T06:20:59.846081+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e129 with expected crc 2023-10-09T06:21:01.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:01.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:01.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:01.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:01.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:01.359 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:01] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-10-09T06:21:01.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:01.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:01.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:01.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:01.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:01.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:01.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:01.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:01.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:01.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:01.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:01 smithi125 bash[77299]: cluster 2023-10-09T06:21:00.732090+0000 mgr.y (mgr.25723) 472 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10566/159141 objects degraded (6.639%); 11241/159141 objects misplaced (7.064%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:21:01.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:01 smithi125 bash[80002]: cluster 2023-10-09T06:21:00.732090+0000 mgr.y (mgr.25723) 472 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10566/159141 objects degraded (6.639%); 11241/159141 objects misplaced (7.064%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:21:01.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:01.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:02.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:02.069 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:01 smithi155 bash[59495]: cluster 2023-10-09T06:21:00.732090+0000 mgr.y (mgr.25723) 472 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10566/159141 objects degraded (6.639%); 11241/159141 objects misplaced (7.064%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:21:02.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:02.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:02.249 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:02.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:02.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:03.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:03.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:03.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:03.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:03.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:03.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:03.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:03.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:03.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:03.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:03.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:03.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:21:03.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:03.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:03.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:03.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:03.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[77299]: cluster 2023-10-09T06:21:02.732824+0000 mgr.y (mgr.25723) 473 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10566/154734 objects degraded (6.828%); 11241/154734 objects misplaced (7.265%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:03.920 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[77299]: audit 2023-10-09T06:21:03.572550+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[77299]: audit 2023-10-09T06:21:03.574750+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[77299]: cluster 2023-10-09T06:21:03.641809+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 10566/154734 objects degraded (6.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[80002]: cluster 2023-10-09T06:21:02.732824+0000 mgr.y (mgr.25723) 473 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10566/154734 objects degraded (6.828%); 11241/154734 objects misplaced (7.265%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[80002]: audit 2023-10-09T06:21:03.572550+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[80002]: audit 2023-10-09T06:21:03.574750+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:03.921 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:03 smithi125 bash[80002]: cluster 2023-10-09T06:21:03.641809+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 10566/154734 objects degraded (6.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:03.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:03.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:04.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:04.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:03 smithi155 bash[59495]: cluster 2023-10-09T06:21:02.732824+0000 mgr.y (mgr.25723) 473 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10566/154734 objects degraded (6.828%); 11241/154734 objects misplaced (7.265%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:04.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:03 smithi155 bash[59495]: audit 2023-10-09T06:21:03.572550+0000 mon.a (mon.0) 645 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:04.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:03 smithi155 bash[59495]: audit 2023-10-09T06:21:03.574750+0000 mon.a (mon.0) 646 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:04.083 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:03 smithi155 bash[59495]: cluster 2023-10-09T06:21:03.641809+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 10566/154734 objects degraded (6.828%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:04.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:04.158 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:04.159 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:04.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:04.244 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:04.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:04.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:04.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:04.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:04.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:04.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:04.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:04.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:04.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:04.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:04.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:04.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:05.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:05.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:05.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:05.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:05.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:05.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:05.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:05.584 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:05.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:05.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:05.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:05 smithi125 bash[80002]: cluster 2023-10-09T06:21:04.733354+0000 mgr.y (mgr.25723) 474 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 10566/152451 objects degraded (6.931%); 11241/152451 objects misplaced (7.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:05.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:05.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:05.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:05.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:06.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:06.110 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:05 smithi125 bash[77299]: cluster 2023-10-09T06:21:04.733354+0000 mgr.y (mgr.25723) 474 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 10566/152451 objects degraded (6.931%); 11241/152451 objects misplaced (7.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:06.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:06.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:05 smithi155 bash[59495]: cluster 2023-10-09T06:21:04.733354+0000 mgr.y (mgr.25723) 474 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 10566/152451 objects degraded (6.931%); 11241/152451 objects misplaced (7.374%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:06.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:06.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:06.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:06.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:06.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:06.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:06.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:06.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:06.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:06.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:07.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:07.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:07.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:07.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:07.217 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:07.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:07.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:07.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:07.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:07.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:07.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:07.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:07.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:07.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:07 smithi125 bash[80002]: cluster 2023-10-09T06:21:06.734031+0000 mgr.y (mgr.25723) 475 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10652/145482 objects degraded (7.322%); 11051/145482 objects misplaced (7.596%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:07.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:07.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:07.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:07.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:07.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:08.006 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:07 smithi155 bash[59495]: cluster 2023-10-09T06:21:06.734031+0000 mgr.y (mgr.25723) 475 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10652/145482 objects degraded (7.322%); 11051/145482 objects misplaced (7.596%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:08.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:08.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:07 smithi125 bash[77299]: cluster 2023-10-09T06:21:06.734031+0000 mgr.y (mgr.25723) 475 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10652/145482 objects degraded (7.322%); 11051/145482 objects misplaced (7.596%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:08.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:08.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:08.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:08.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:08.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:08.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:08.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:08.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:08.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:08.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:08.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:08.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:08 smithi125 bash[77299]: cluster 2023-10-09T06:21:08.643407+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 10652/145482 objects degraded (7.322%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:08.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:08.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:08.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:09.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:09.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:09.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:08 smithi125 bash[80002]: cluster 2023-10-09T06:21:08.643407+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 10652/145482 objects degraded (7.322%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:09.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:08 smithi155 bash[59495]: cluster 2023-10-09T06:21:08.643407+0000 mon.a (mon.0) 648 : cluster 3 Health check update: Degraded data redundancy: 10652/145482 objects degraded (7.322%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:09.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:09.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:09.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:09.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:09.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:09.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:09.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:09.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:09.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:09.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:09.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:09.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:09.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:09.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:09 smithi125 bash[77299]: cluster 2023-10-09T06:21:08.734715+0000 mgr.y (mgr.25723) 476 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10652/143832 objects degraded (7.406%); 11051/143832 objects misplaced (7.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:09.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:09 smithi125 bash[80002]: cluster 2023-10-09T06:21:08.734715+0000 mgr.y (mgr.25723) 476 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10652/143832 objects degraded (7.406%); 11051/143832 objects misplaced (7.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:09.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:09.965 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:09 smithi155 bash[59495]: cluster 2023-10-09T06:21:08.734715+0000 mgr.y (mgr.25723) 476 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10652/143832 objects degraded (7.406%); 11051/143832 objects misplaced (7.683%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:09.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:09.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:10.073 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:10.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:10.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:10.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:10.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:10.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:10.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:10.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:10.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:10.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:10.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:10.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:10.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:10.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:10.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:10.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:10.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:11.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:11.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:11.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:11.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:11.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:11.468 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:21:11.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:11.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:11.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:11.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:11.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:11.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:11.874 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:11 smithi125 bash[77299]: cluster 2023-10-09T06:21:10.735377+0000 mgr.y (mgr.25723) 477 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 10732/141159 objects degraded (7.603%); 10872/141159 objects misplaced (7.702%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:21:11.944 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:11.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:11.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:12.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:12.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:12.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:11 smithi125 bash[80002]: cluster 2023-10-09T06:21:10.735377+0000 mgr.y (mgr.25723) 477 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 10732/141159 objects degraded (7.603%); 10872/141159 objects misplaced (7.702%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:21:12.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:11 smithi155 bash[59495]: cluster 2023-10-09T06:21:10.735377+0000 mgr.y (mgr.25723) 477 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 10732/141159 objects degraded (7.603%); 10872/141159 objects misplaced (7.702%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:21:12.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:12.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:12.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:12.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:12.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:12.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:12.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:12.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:12.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:12.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:12.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:12.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:13.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:13.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:13.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:13.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:13.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:13.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:13.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:13.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:13.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:13.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:13.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:13.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:13.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:13.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:13.848 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:13.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:13.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:13.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:14.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:14.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:13 smithi125 bash[77299]: cluster 2023-10-09T06:21:12.736028+0000 mgr.y (mgr.25723) 478 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10732/136746 objects degraded (7.848%); 10872/136746 objects misplaced (7.951%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:14.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:13 smithi125 bash[77299]: cluster 2023-10-09T06:21:13.644568+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 10732/136746 objects degraded (7.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:14.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:13 smithi125 bash[80002]: cluster 2023-10-09T06:21:12.736028+0000 mgr.y (mgr.25723) 478 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10732/136746 objects degraded (7.848%); 10872/136746 objects misplaced (7.951%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:14.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:13 smithi125 bash[80002]: cluster 2023-10-09T06:21:13.644568+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 10732/136746 objects degraded (7.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:13 smithi155 bash[59495]: cluster 2023-10-09T06:21:12.736028+0000 mgr.y (mgr.25723) 478 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 10732/136746 objects degraded (7.848%); 10872/136746 objects misplaced (7.951%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:13 smithi155 bash[59495]: cluster 2023-10-09T06:21:13.644568+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 10732/136746 objects degraded (7.848%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:14.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:14.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:14.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:14.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:14.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:14.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:14.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:14.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:14.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:14.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:14.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:14.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:14.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:14.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:15.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:15.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:15.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:15.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:15.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:15.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:15.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:15.520 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:15.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:15.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:15.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:15.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:15.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:15 smithi155 bash[59495]: cluster 2023-10-09T06:21:14.736656+0000 mgr.y (mgr.25723) 479 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10732/134523 objects degraded (7.978%); 10872/134523 objects misplaced (8.082%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:15.883 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:15 smithi155 bash[59495]: audit 2023-10-09T06:21:15.462624+0000 mon.a (mon.0) 650 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:15.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:15.892 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:21:15.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:15.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:16.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:16.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:15 smithi125 bash[80002]: cluster 2023-10-09T06:21:14.736656+0000 mgr.y (mgr.25723) 479 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10732/134523 objects degraded (7.978%); 10872/134523 objects misplaced (8.082%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:15 smithi125 bash[80002]: audit 2023-10-09T06:21:15.462624+0000 mon.a (mon.0) 650 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:15 smithi125 bash[77299]: cluster 2023-10-09T06:21:14.736656+0000 mgr.y (mgr.25723) 479 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10732/134523 objects degraded (7.978%); 10872/134523 objects misplaced (8.082%); 1.1 MiB/s, 16 objects/s recovering 2023-10-09T06:21:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:15 smithi125 bash[77299]: audit 2023-10-09T06:21:15.462624+0000 mon.a (mon.0) 650 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:21:16.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:16.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:16.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:16.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:16.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:16.482 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:16.482 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:16.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:16.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:16.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:16.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:16.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:16.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:16.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:16.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:16.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:16.991 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: audit 2023-10-09T06:21:15.869180+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:16.991 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.869252+0000 mon.a (mon.0) 652 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:21:16.991 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.874947+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.991 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.877964+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.878894+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.879998+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.880354+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.882284+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.882909+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:16.992 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:16 smithi155 bash[59495]: cluster 2023-10-09T06:21:15.886662+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:17.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:17.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:17.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: audit 2023-10-09T06:21:15.869180+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:17.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.869252+0000 mon.a (mon.0) 652 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:21:17.150 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.874947+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.877964+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.878894+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.879998+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.880354+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.882284+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.882909+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[77299]: cluster 2023-10-09T06:21:15.886662+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.151 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: audit 2023-10-09T06:21:15.869180+0000 mon.a (mon.0) 651 : audit 1 from='client.? 172.21.15.125:0/362533166' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.869252+0000 mon.a (mon.0) 652 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.874947+0000 osd.4 (osd.4) 58 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.877964+0000 osd.4 (osd.4) 59 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.878894+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.879998+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.880354+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.882284+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.882909+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.152 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:16 smithi125 bash[80002]: cluster 2023-10-09T06:21:15.886662+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e130 with expected crc 2023-10-09T06:21:17.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:17.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:17.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:17.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:17.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:17.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:17.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:17.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:17.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:17.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.937 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:17 smithi155 bash[59495]: cluster 2023-10-09T06:21:16.737313+0000 mgr.y (mgr.25723) 480 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10847/127515 objects degraded (8.506%); 10657/127515 objects misplaced (8.357%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:17.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:17.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:18.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:18.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:17 smithi125 bash[77299]: cluster 2023-10-09T06:21:16.737313+0000 mgr.y (mgr.25723) 480 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10847/127515 objects degraded (8.506%); 10657/127515 objects misplaced (8.357%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:18.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:17 smithi125 bash[80002]: cluster 2023-10-09T06:21:16.737313+0000 mgr.y (mgr.25723) 480 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10847/127515 objects degraded (8.506%); 10657/127515 objects misplaced (8.357%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:18.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:18.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:18.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:18.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:21:18.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:18.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:18.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:18.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:18.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:18.804 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:18.910 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:18 smithi125 bash[77299]: audit 2023-10-09T06:21:18.568495+0000 mon.a (mon.0) 653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:18.910 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:18 smithi125 bash[77299]: cluster 2023-10-09T06:21:18.645730+0000 mon.a (mon.0) 654 : cluster 3 Health check update: Degraded data redundancy: 10847/127515 objects degraded (8.506%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:18.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:18 smithi125 bash[80002]: audit 2023-10-09T06:21:18.568495+0000 mon.a (mon.0) 653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:18.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:18 smithi125 bash[80002]: cluster 2023-10-09T06:21:18.645730+0000 mon.a (mon.0) 654 : cluster 3 Health check update: Degraded data redundancy: 10847/127515 objects degraded (8.506%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:18.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:18.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:19.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:19.094 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:19.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:19.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:19.209 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:18 smithi155 bash[59495]: audit 2023-10-09T06:21:18.568495+0000 mon.a (mon.0) 653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:19.210 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:18 smithi155 bash[59495]: cluster 2023-10-09T06:21:18.645730+0000 mon.a (mon.0) 654 : cluster 3 Health check update: Degraded data redundancy: 10847/127515 objects degraded (8.506%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:19.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:19.265 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:21:19.300 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:21:19.339 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:21:19.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:19.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:19.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:19.386 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:21:19.433 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:21:19.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:19.484 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:21:19.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:19.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:19.520 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:21:19.551 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.559 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:19.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:19.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:19.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:19.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:19.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:19.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:19.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:20.023 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:20.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:19 smithi155 bash[59495]: cluster 2023-10-09T06:21:18.737867+0000 mgr.y (mgr.25723) 481 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10847/125907 objects degraded (8.615%); 10657/125907 objects misplaced (8.464%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:20.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:19 smithi155 bash[59495]: audit 2023-10-09T06:21:19.226449+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.125:0/398789164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:20.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:20.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:19 smithi125 bash[77299]: cluster 2023-10-09T06:21:18.737867+0000 mgr.y (mgr.25723) 481 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10847/125907 objects degraded (8.615%); 10657/125907 objects misplaced (8.464%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:20.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:19 smithi125 bash[77299]: audit 2023-10-09T06:21:19.226449+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.125:0/398789164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:20.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:19 smithi125 bash[80002]: cluster 2023-10-09T06:21:18.737867+0000 mgr.y (mgr.25723) 481 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10847/125907 objects degraded (8.615%); 10657/125907 objects misplaced (8.464%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:21:20.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:19 smithi125 bash[80002]: audit 2023-10-09T06:21:19.226449+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.125:0/398789164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:20.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:21:20.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:20.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:20.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:20.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:20.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:20.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:20.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:20.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:21.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:21.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:21.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:21.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:21.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:21.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:21] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:21:21.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:21.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:21.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:21.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:21.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:22.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:22.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:21 smithi125 bash[77299]: cluster 2023-10-09T06:21:20.739183+0000 mgr.y (mgr.25723) 482 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 10920/123264 objects degraded (8.859%); 10478/123264 objects misplaced (8.500%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:22.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:21 smithi125 bash[80002]: cluster 2023-10-09T06:21:20.739183+0000 mgr.y (mgr.25723) 482 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 10920/123264 objects degraded (8.859%); 10478/123264 objects misplaced (8.500%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:22.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:22.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:21 smithi155 bash[59495]: cluster 2023-10-09T06:21:20.739183+0000 mgr.y (mgr.25723) 482 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 10920/123264 objects degraded (8.859%); 10478/123264 objects misplaced (8.500%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:22.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:22.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:22.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:22.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:22.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:22.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:22.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:22.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:22.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.929 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:21:22.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:22.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:22.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:23.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:23.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:22 smithi125 bash[77299]: audit 2023-10-09T06:21:22.670220+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.125:0/2954740296' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:22 smithi125 bash[77299]: audit 2023-10-09T06:21:22.670680+0000 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:22 smithi125 bash[80002]: audit 2023-10-09T06:21:22.670220+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.125:0/2954740296' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.070 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:22 smithi125 bash[80002]: audit 2023-10-09T06:21:22.670680+0000 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.087 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:23.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:22 smithi155 bash[59495]: audit 2023-10-09T06:21:22.670220+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.125:0/2954740296' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:22 smithi155 bash[59495]: audit 2023-10-09T06:21:22.670680+0000 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:23.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.241 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:23.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:23.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:23.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:23.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:23.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:23.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:23.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:23.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:23.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:23.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:23.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:24.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:24.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:24.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.739843+0000 mgr.y (mgr.25723) 483 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10920/118728 objects degraded (9.197%); 10478/118728 objects misplaced (8.825%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.901228+0000 mon.a (mon.0) 656 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: audit 2023-10-09T06:21:22.914680+0000 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.914736+0000 mon.a (mon.0) 658 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.915853+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.929992+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.932000+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.162 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.941943+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:22.943730+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.163 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:23 smithi155 bash[59495]: cluster 2023-10-09T06:21:23.646921+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 10920/118728 objects degraded (9.197%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:24.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:24.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.739843+0000 mgr.y (mgr.25723) 483 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10920/118728 objects degraded (9.197%); 10478/118728 objects misplaced (8.825%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.901228+0000 mon.a (mon.0) 656 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: audit 2023-10-09T06:21:22.914680+0000 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.914736+0000 mon.a (mon.0) 658 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.915853+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.929992+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.932000+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.941943+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:22.943730+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[77299]: cluster 2023-10-09T06:21:23.646921+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 10920/118728 objects degraded (9.197%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.739843+0000 mgr.y (mgr.25723) 483 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10920/118728 objects degraded (9.197%); 10478/118728 objects misplaced (8.825%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.901228+0000 mon.a (mon.0) 656 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:24.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: audit 2023-10-09T06:21:22.914680+0000 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.914736+0000 mon.a (mon.0) 658 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.915853+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.929992+0000 osd.4 (osd.4) 60 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.932000+0000 osd.4 (osd.4) 61 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.941943+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:22.943730+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-09T06:21:24.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:23 smithi125 bash[80002]: cluster 2023-10-09T06:21:23.646921+0000 mon.a (mon.0) 659 : cluster 3 Health check update: Degraded data redundancy: 10920/118728 objects degraded (9.197%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:24.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:24.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:24.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:24.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:24.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:24.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:24.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:24.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:24.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:24.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:24.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:25.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:25.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:25.129 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:25.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:25.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:25.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:25.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:25.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:25.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:25.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:25.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:25.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:25.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:25.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:25.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:25.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:25.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:25.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:26.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:26.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:26.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:26.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:25 smithi155 bash[59495]: cluster 2023-10-09T06:21:24.740470+0000 mgr.y (mgr.25723) 484 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10920/116622 objects degraded (9.364%); 10478/116622 objects misplaced (8.985%); 765 KiB/s, 11 objects/s recovering 2023-10-09T06:21:26.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:21:26.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:26.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:26.196 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:25 smithi125 bash[77299]: cluster 2023-10-09T06:21:24.740470+0000 mgr.y (mgr.25723) 484 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10920/116622 objects degraded (9.364%); 10478/116622 objects misplaced (8.985%); 765 KiB/s, 11 objects/s recovering 2023-10-09T06:21:26.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:26 smithi125 bash[80002]: cluster 2023-10-09T06:21:24.740470+0000 mgr.y (mgr.25723) 484 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10920/116622 objects degraded (9.364%); 10478/116622 objects misplaced (8.985%); 765 KiB/s, 11 objects/s recovering 2023-10-09T06:21:26.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:26.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:26.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:26.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:26.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:26.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:26.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:26.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:26.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:26.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:26.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:26.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:26.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:26.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:27.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:27.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:27.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:27.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:27.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:27.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:27.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:27.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:27.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:27.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:27.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:28.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:28.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:28.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:28.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:27 smithi125 bash[77299]: cluster 2023-10-09T06:21:26.741273+0000 mgr.y (mgr.25723) 485 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11001/109998 objects degraded (10.001%); 10292/109998 objects misplaced (9.357%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:28.191 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:28 smithi125 bash[80002]: cluster 2023-10-09T06:21:26.741273+0000 mgr.y (mgr.25723) 485 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11001/109998 objects degraded (10.001%); 10292/109998 objects misplaced (9.357%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:28.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:28.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:28.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:28.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:27 smithi155 bash[59495]: cluster 2023-10-09T06:21:26.741273+0000 mgr.y (mgr.25723) 485 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 11001/109998 objects degraded (10.001%); 10292/109998 objects misplaced (9.357%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:28.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:28.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:28.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:28.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:28.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:28.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:28.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:28.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:28.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:28.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:29.058 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:29.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:29.135 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:28 smithi155 bash[59495]: cluster 2023-10-09T06:21:28.648362+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 11001/109998 objects degraded (10.001%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:29.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:29.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.280 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:28 smithi125 bash[80002]: cluster 2023-10-09T06:21:28.648362+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 11001/109998 objects degraded (10.001%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:29.281 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:28 smithi125 bash[77299]: cluster 2023-10-09T06:21:28.648362+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 11001/109998 objects degraded (10.001%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:29.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:29.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:29.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:29.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:29.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:29.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:29.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:29.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:29.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:29.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:29.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:29.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:30.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:30.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:30.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:30.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:29 smithi125 bash[77299]: cluster 2023-10-09T06:21:28.741794+0000 mgr.y (mgr.25723) 486 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 11001/108123 objects degraded (10.175%); 10292/108123 objects misplaced (9.519%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:30.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:29 smithi125 bash[80002]: cluster 2023-10-09T06:21:28.741794+0000 mgr.y (mgr.25723) 486 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 11001/108123 objects degraded (10.175%); 10292/108123 objects misplaced (9.519%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:30.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:30.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:30.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:30.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:29 smithi155 bash[59495]: cluster 2023-10-09T06:21:28.741794+0000 mgr.y (mgr.25723) 486 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 11001/108123 objects degraded (10.175%); 10292/108123 objects misplaced (9.519%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:30.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:30.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:30.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:30.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:30.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:30.731 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:30.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:30.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:30.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:30.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:30.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:31.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:31.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:31.123 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:21:31.123 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:21:31.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.123 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:174411 5.f e131/105 5:f6d56d0a:::benchmark_data_smithi125_106927_object70749:head v 131'11009, mlcod=131'11006)", 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:31.108214+0000", 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.014150962, 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014181701, 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108214+0000", 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:21:31.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108214+0000", 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108213+0000", 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7840000000000004e-06 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.125 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108221+0000", 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0835e-05 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108232+0000", 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0659999999999998e-06 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108239+0000", 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000156749 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108396+0000", 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5312000000000001e-05 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:21:31.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108461+0000", 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024666400000000001 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:174416 5.1 5:8e135e5f:::benchmark_data_smithi125_106927_object70754:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:31.108363+0000", 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.014002667, 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014106518, 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:21:31.128 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "delayed", 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 174416 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108363+0000", 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:21:31.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108363+0000", 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108361+0000", 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1130000000000003e-06 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108368+0000", 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0672e-05 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108379+0000", 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0020000000000003e-06 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:21:31.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.108385+0000", 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011712820000000001 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.109556+0000", 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000144363 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for rw locks", 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.109700+0000", 2023-10-09T06:21:31.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00133759 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:174417 5.f e131/105 5:fc90c5a3:::benchmark_data_smithi125_106927_object70755:head v 131'11010, mlcod=131'11006)", 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:31.116757+0000", 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0056081949999999998, 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057854109999999999, 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:21:31.133 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116757+0000", 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116757+0000", 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116756+0000", 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2199999999999992e-06 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116764+0000", 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4347e-05 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116779+0000", 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6429999999999998e-06 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.116785+0000", 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00087151000000000004 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117657+0000", 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4653e-05 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117711+0000", 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00095430099999999996 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:174425 5.e 5:7019ac9a:::benchmark_data_smithi125_106927_object70763:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:31.117952+0000", 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0044134980000000001, 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046539249999999997, 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 174425 2023-10-09T06:21:31.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.139 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:21:31.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:21:31.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117952+0000", 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117952+0000", 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:31.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117950+0000", 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4229999999999999e-06 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117958+0000", 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4284000000000001e-05 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117972+0000", 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4489999999999996e-06 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.117978+0000", 2023-10-09T06:21:31.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9987000000000001e-05 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.118028+0000", 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013605099999999999 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.118164+0000", 2023-10-09T06:21:31.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013819400000000001 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.118303+0000", 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035065999999999998 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.144 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPushReply(5.1 131/118 [PushReplyOp(5:8e135e5f:::benchmark_data_smithi125_106927_object70754:head)])", 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:31.119837+0000", 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.002528176, 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028434710000000002, 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:21:31.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119837+0000", 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119837+0000", 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:21:31.146 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119837+0000", 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0990000000000001e-06 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119841+0000", 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9789999999999996e-06 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:21:31.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119848+0000", 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7299999999999996e-06 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.119853+0000", 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015292229999999999 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:21:31.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:31.121382+0000", 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015447049999999999 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 5 2023-10-09T06:21:31.149 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:21:31.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:31.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:31.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:31.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:31.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:31.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:31.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:31.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:31.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:31.601 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:31] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:21:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:31.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:31.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:31.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:31.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:31.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:31.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:31.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:31.995 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:31 smithi155 bash[59495]: cluster 2023-10-09T06:21:30.742485+0000 mgr.y (mgr.25723) 487 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 11107/105093 objects degraded (10.569%); 10077/105093 objects misplaced (9.589%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:32.012 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:32.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:21:32.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:32.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:32.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:31 smithi125 bash[77299]: cluster 2023-10-09T06:21:30.742485+0000 mgr.y (mgr.25723) 487 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 11107/105093 objects degraded (10.569%); 10077/105093 objects misplaced (9.589%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:32.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:31 smithi125 bash[80002]: cluster 2023-10-09T06:21:30.742485+0000 mgr.y (mgr.25723) 487 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 11107/105093 objects degraded (10.569%); 10077/105093 objects misplaced (9.589%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:32.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:32.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:32.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:32.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:32.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:32.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:32.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:32.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:32.961 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:21:32.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:32.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:33.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:33.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:33.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:33.238 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:32 smithi155 bash[59495]: audit 2023-10-09T06:21:32.642646+0000 mon.a (mon.0) 661 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:33.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:33.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:32 smithi125 bash[77299]: audit 2023-10-09T06:21:32.642646+0000 mon.a (mon.0) 661 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:32 smithi125 bash[80002]: audit 2023-10-09T06:21:32.642646+0000 mon.a (mon.0) 661 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:33.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:33.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:33.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:33.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:33.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:33.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:33.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:33.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:33.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:33.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:34.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: cluster 2023-10-09T06:21:32.743179+0000 mgr.y (mgr.25723) 488 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11107/100584 objects degraded (11.043%); 10077/100584 objects misplaced (10.018%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: audit 2023-10-09T06:21:32.949805+0000 mon.a (mon.0) 662 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: cluster 2023-10-09T06:21:32.949885+0000 mon.a (mon.0) 663 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: cluster 2023-10-09T06:21:32.950952+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: audit 2023-10-09T06:21:33.568609+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:34.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[77299]: cluster 2023-10-09T06:21:33.649528+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 11107/100584 objects degraded (11.043%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: cluster 2023-10-09T06:21:32.743179+0000 mgr.y (mgr.25723) 488 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11107/100584 objects degraded (11.043%); 10077/100584 objects misplaced (10.018%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: audit 2023-10-09T06:21:32.949805+0000 mon.a (mon.0) 662 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: cluster 2023-10-09T06:21:32.949885+0000 mon.a (mon.0) 663 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: cluster 2023-10-09T06:21:32.950952+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: audit 2023-10-09T06:21:33.568609+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:34.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:33 smithi125 bash[80002]: cluster 2023-10-09T06:21:33.649528+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 11107/100584 objects degraded (11.043%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:34.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:34.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:34.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:34.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:34.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: cluster 2023-10-09T06:21:32.743179+0000 mgr.y (mgr.25723) 488 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11107/100584 objects degraded (11.043%); 10077/100584 objects misplaced (10.018%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: audit 2023-10-09T06:21:32.949805+0000 mon.a (mon.0) 662 : audit 1 from='client.? 172.21.15.125:0/3663225387' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: cluster 2023-10-09T06:21:32.949885+0000 mon.a (mon.0) 663 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: cluster 2023-10-09T06:21:32.950952+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e132 with expected crc 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: audit 2023-10-09T06:21:33.568609+0000 mon.a (mon.0) 664 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:34.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:33 smithi155 bash[59495]: cluster 2023-10-09T06:21:33.649528+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 11107/100584 objects degraded (11.043%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:34.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:34.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:34.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:34.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:34.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:34.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:34.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:34.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:34.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:34.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:34.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:35.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:35.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:35.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:35.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:35.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:35.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:35.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:35.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:35.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:35.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:35.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:35.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:35.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:35.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:35.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:35.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:21:35.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:36.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:35 smithi155 bash[59495]: cluster 2023-10-09T06:21:34.743768+0000 mgr.y (mgr.25723) 489 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11107/98124 objects degraded (11.319%); 10077/98124 objects misplaced (10.270%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:36.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:36.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:36.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:36.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:36.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:35 smithi125 bash[77299]: cluster 2023-10-09T06:21:34.743768+0000 mgr.y (mgr.25723) 489 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11107/98124 objects degraded (11.319%); 10077/98124 objects misplaced (10.270%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:36.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:35 smithi125 bash[80002]: cluster 2023-10-09T06:21:34.743768+0000 mgr.y (mgr.25723) 489 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11107/98124 objects degraded (11.319%); 10077/98124 objects misplaced (10.270%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:36.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:36.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:36.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:36.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:36.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:36.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:36.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:36.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:36.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:36.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:36.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:36.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:36.993 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:37.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:37.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:37.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:37.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:37.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:37.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:37.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:37.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:37.668 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:37.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:37.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:37.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:37.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:37.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:37.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:38.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:38.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:37 smithi125 bash[77299]: cluster 2023-10-09T06:21:36.744703+0000 mgr.y (mgr.25723) 490 : cluster 0 pgmap v450: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11182/91302 objects degraded (12.247%); 9895/91302 objects misplaced (10.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:38.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:37 smithi125 bash[80002]: cluster 2023-10-09T06:21:36.744703+0000 mgr.y (mgr.25723) 490 : cluster 0 pgmap v450: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11182/91302 objects degraded (12.247%); 9895/91302 objects misplaced (10.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:38.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:38.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:38.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:38.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:37 smithi155 bash[59495]: cluster 2023-10-09T06:21:36.744703+0000 mgr.y (mgr.25723) 490 : cluster 0 pgmap v450: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11182/91302 objects degraded (12.247%); 9895/91302 objects misplaced (10.838%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:38.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:38.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:38.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:38.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:38.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:38.787 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:38.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:38.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:38.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:38.966 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:38 smithi125 bash[77299]: cluster 2023-10-09T06:21:38.650446+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 11182/91302 objects degraded (12.247%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:39.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:39.039 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:38 smithi125 bash[80002]: cluster 2023-10-09T06:21:38.650446+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 11182/91302 objects degraded (12.247%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:39.044 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:21:39.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:39.091 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:21:39.126 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:21:39.160 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:21:39.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:39.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:39.207 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:21:39.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:39.243 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:21:39.287 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:21:39.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:39.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:38 smithi155 bash[59495]: cluster 2023-10-09T06:21:38.650446+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 11182/91302 objects degraded (12.247%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:39.330 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.338 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:39.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:39.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:39.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:39.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:39.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:39.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:39.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:39.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:39.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:39.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:39.987 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:21:40.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:40.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: cluster 2023-10-09T06:21:38.745401+0000 mgr.y (mgr.25723) 491 : cluster 0 pgmap v451: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 11182/89697 objects degraded (12.466%); 9895/89697 objects misplaced (11.032%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:40.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: audit 2023-10-09T06:21:38.970685+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.125:0/3719278527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:40.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: audit 2023-10-09T06:21:39.339302+0000 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:40.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: cluster 2023-10-09T06:21:38.745401+0000 mgr.y (mgr.25723) 491 : cluster 0 pgmap v451: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 11182/89697 objects degraded (12.466%); 9895/89697 objects misplaced (11.032%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:40.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: audit 2023-10-09T06:21:38.970685+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.125:0/3719278527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:40.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: audit 2023-10-09T06:21:39.339302+0000 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:40.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:40.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:40.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.302 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:40.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:39 smithi155 bash[59495]: cluster 2023-10-09T06:21:38.745401+0000 mgr.y (mgr.25723) 491 : cluster 0 pgmap v451: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 11182/89697 objects degraded (12.466%); 9895/89697 objects misplaced (11.032%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:40.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:39 smithi155 bash[59495]: audit 2023-10-09T06:21:38.970685+0000 mon.a (mon.0) 667 : audit 0 from='client.? 172.21.15.125:0/3719278527' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:40.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:39 smithi155 bash[59495]: audit 2023-10-09T06:21:39.339302+0000 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:40.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:40.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:40.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:40.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.710 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:40.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:40.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:40.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:40.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:40.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:40.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:41.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:41.085 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: cluster 2023-10-09T06:21:39.960528+0000 mon.a (mon.0) 669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:41.085 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: audit 2023-10-09T06:21:39.977160+0000 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:41.085 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: cluster 2023-10-09T06:21:39.977841+0000 mon.a (mon.0) 671 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:21:41.085 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: cluster 2023-10-09T06:21:39.978217+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.085 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[77299]: cluster 2023-10-09T06:21:39.979849+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: cluster 2023-10-09T06:21:39.960528+0000 mon.a (mon.0) 669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:41.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: audit 2023-10-09T06:21:39.977160+0000 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:41.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: cluster 2023-10-09T06:21:39.977841+0000 mon.a (mon.0) 671 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:21:41.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: cluster 2023-10-09T06:21:39.978217+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.086 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:40 smithi125 bash[80002]: cluster 2023-10-09T06:21:39.979849+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:41.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:41.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:41.242 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:41] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-10-09T06:21:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:41.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:40 smithi155 bash[59495]: cluster 2023-10-09T06:21:39.960528+0000 mon.a (mon.0) 669 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:41.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:40 smithi155 bash[59495]: audit 2023-10-09T06:21:39.977160+0000 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.125:0/3233303904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:41.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:40 smithi155 bash[59495]: cluster 2023-10-09T06:21:39.977841+0000 mon.a (mon.0) 671 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:21:41.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:40 smithi155 bash[59495]: cluster 2023-10-09T06:21:39.978217+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:40 smithi155 bash[59495]: cluster 2023-10-09T06:21:39.979849+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-10-09T06:21:41.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:41.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:41.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:41.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:41.535 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:41.536 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:41.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:41.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:41.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:41.788 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:41.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:41.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:41.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:42.030 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:41 smithi125 bash[77299]: cluster 2023-10-09T06:21:40.745993+0000 mgr.y (mgr.25723) 492 : cluster 0 pgmap v453: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11254/86892 objects degraded (12.952%); 9714/86892 objects misplaced (11.179%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:21:42.030 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:41 smithi125 bash[77299]: audit 2023-10-09T06:21:41.727486+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.030 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:41 smithi125 bash[80002]: cluster 2023-10-09T06:21:40.745993+0000 mgr.y (mgr.25723) 492 : cluster 0 pgmap v453: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11254/86892 objects degraded (12.952%); 9714/86892 objects misplaced (11.179%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:21:42.031 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:41 smithi125 bash[80002]: audit 2023-10-09T06:21:41.727486+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:42.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:42.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:41 smithi155 bash[59495]: cluster 2023-10-09T06:21:40.745993+0000 mgr.y (mgr.25723) 492 : cluster 0 pgmap v453: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 11254/86892 objects degraded (12.952%); 9714/86892 objects misplaced (11.179%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:21:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:41 smithi155 bash[59495]: audit 2023-10-09T06:21:41.727486+0000 mon.a (mon.0) 672 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:42.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:42.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:42.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:42.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:42.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:42.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:42.579 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:42.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:21:42.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:42.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:42.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:42.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:42.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:42.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:42.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:42.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:43.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:43.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:43.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:43.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:43.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:43.359 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:43.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:43.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:43.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:43.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:43.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:43.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:43.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:43.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:43.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:44.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:44.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:44.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:44.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:21:44.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:43 smithi125 bash[80002]: cluster 2023-10-09T06:21:42.748547+0000 mgr.y (mgr.25723) 493 : cluster 0 pgmap v454: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11254/82236 objects degraded (13.685%); 9714/82236 objects misplaced (11.812%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:21:44.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:43 smithi125 bash[80002]: cluster 2023-10-09T06:21:43.651734+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11254/82236 objects degraded (13.685%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:44.106 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:43 smithi125 bash[77299]: cluster 2023-10-09T06:21:42.748547+0000 mgr.y (mgr.25723) 493 : cluster 0 pgmap v454: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11254/82236 objects degraded (13.685%); 9714/82236 objects misplaced (11.812%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:21:44.106 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:43 smithi125 bash[77299]: cluster 2023-10-09T06:21:43.651734+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11254/82236 objects degraded (13.685%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:44.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:44.138 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:43 smithi155 bash[59495]: cluster 2023-10-09T06:21:42.748547+0000 mgr.y (mgr.25723) 493 : cluster 0 pgmap v454: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 11254/82236 objects degraded (13.685%); 9714/82236 objects misplaced (11.812%); 1.4 MiB/s, 22 objects/s recovering 2023-10-09T06:21:44.138 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:43 smithi155 bash[59495]: cluster 2023-10-09T06:21:43.651734+0000 mon.a (mon.0) 673 : cluster 3 Health check update: Degraded data redundancy: 11254/82236 objects degraded (13.685%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:44.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:44.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:44.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:44.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:44.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:44.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:44.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:44.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:44.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:44.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:44.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:44.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:45.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:45.129 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:45.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:45.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:45.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:45.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:45.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:45.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:45.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:45.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:45.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:45.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:45.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:46.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:46.081 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:45 smithi155 bash[59495]: cluster 2023-10-09T06:21:44.749217+0000 mgr.y (mgr.25723) 494 : cluster 0 pgmap v455: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11254/79932 objects degraded (14.079%); 9714/79932 objects misplaced (12.153%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:46.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:46.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:46.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:46.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:46.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:46.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:45 smithi125 bash[77299]: cluster 2023-10-09T06:21:44.749217+0000 mgr.y (mgr.25723) 494 : cluster 0 pgmap v455: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11254/79932 objects degraded (14.079%); 9714/79932 objects misplaced (12.153%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:45 smithi125 bash[80002]: cluster 2023-10-09T06:21:44.749217+0000 mgr.y (mgr.25723) 494 : cluster 0 pgmap v455: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11254/79932 objects degraded (14.079%); 9714/79932 objects misplaced (12.153%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:21:46.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:46.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:46.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:46.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:46.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:46.714 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:46.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:46.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:46.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:46.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:46.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:47.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:47.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:47.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:47.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:47.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:47.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.434 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:47.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:47.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:47.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:47.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:47.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:48.018 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:21:48.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:48.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:48 smithi155 bash[59495]: cluster 2023-10-09T06:21:46.750144+0000 mgr.y (mgr.25723) 495 : cluster 0 pgmap v456: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11182/73107 objects degraded (15.295%); 9679/73107 objects misplaced (13.239%); 851 KiB/s, 21 objects/s recovering 2023-10-09T06:21:48.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:48 smithi155 bash[59495]: audit 2023-10-09T06:21:47.066402+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:21:48.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:48.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:48.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:47 smithi125 bash[77299]: cluster 2023-10-09T06:21:46.750144+0000 mgr.y (mgr.25723) 495 : cluster 0 pgmap v456: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11182/73107 objects degraded (15.295%); 9679/73107 objects misplaced (13.239%); 851 KiB/s, 21 objects/s recovering 2023-10-09T06:21:48.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:47 smithi125 bash[77299]: audit 2023-10-09T06:21:47.066402+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:21:48.299 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:47 smithi125 bash[80002]: cluster 2023-10-09T06:21:46.750144+0000 mgr.y (mgr.25723) 495 : cluster 0 pgmap v456: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 11182/73107 objects degraded (15.295%); 9679/73107 objects misplaced (13.239%); 851 KiB/s, 21 objects/s recovering 2023-10-09T06:21:48.299 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:47 smithi125 bash[80002]: audit 2023-10-09T06:21:47.066402+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:21:48.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:48.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:48.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:48.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:48.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:48.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:48.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:48.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:48.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:48.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:49.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:49.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: audit 2023-10-09T06:21:48.005050+0000 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:21:49.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.005125+0000 mon.a (mon.0) 676 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.014273+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.015027+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.015408+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.015486+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.017637+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.024512+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: audit 2023-10-09T06:21:48.569518+0000 mon.a (mon.0) 677 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: audit 2023-10-09T06:21:48.580383+0000 mon.a (mon.0) 678 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: audit 2023-10-09T06:21:48.594945+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.652905+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 11182/73107 objects degraded (15.295%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: audit 2023-10-09T06:21:48.005050+0000 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.005125+0000 mon.a (mon.0) 676 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.014273+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.015027+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.015408+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.015486+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.017637+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.024512+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: audit 2023-10-09T06:21:48.569518+0000 mon.a (mon.0) 677 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:49.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: audit 2023-10-09T06:21:48.580383+0000 mon.a (mon.0) 678 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: audit 2023-10-09T06:21:48.594945+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:49 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.652905+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 11182/73107 objects degraded (15.295%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:49.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:49.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:49.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:49.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: audit 2023-10-09T06:21:48.005050+0000 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.125:0/1912501237' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.005125+0000 mon.a (mon.0) 676 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.014273+0000 osd.4 (osd.4) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.015027+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.015408+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.015486+0000 osd.4 (osd.4) 63 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.017637+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.024512+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e134 with expected crc 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: audit 2023-10-09T06:21:48.569518+0000 mon.a (mon.0) 677 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: audit 2023-10-09T06:21:48.580383+0000 mon.a (mon.0) 678 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: audit 2023-10-09T06:21:48.594945+0000 mon.a (mon.0) 679 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:49 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.652905+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 11182/73107 objects degraded (15.295%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:49.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:21:49.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:49.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:49.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:49.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:49.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:49.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:49.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:49.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:49.897 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:49.897 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:49.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:50.019 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:50 smithi155 bash[59495]: cluster 2023-10-09T06:21:48.750646+0000 mgr.y (mgr.25723) 496 : cluster 0 pgmap v458: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11182/71310 objects degraded (15.681%); 9679/71310 objects misplaced (13.573%); 175 KiB/s, 12 objects/s recovering 2023-10-09T06:21:50.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:50.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:50 smithi125 bash[77299]: cluster 2023-10-09T06:21:48.750646+0000 mgr.y (mgr.25723) 496 : cluster 0 pgmap v458: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11182/71310 objects degraded (15.681%); 9679/71310 objects misplaced (13.573%); 175 KiB/s, 12 objects/s recovering 2023-10-09T06:21:50.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:50 smithi125 bash[80002]: cluster 2023-10-09T06:21:48.750646+0000 mgr.y (mgr.25723) 496 : cluster 0 pgmap v458: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11182/71310 objects degraded (15.681%); 9679/71310 objects misplaced (13.573%); 175 KiB/s, 12 objects/s recovering 2023-10-09T06:21:50.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:50.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:50.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:50.298 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:50.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:50.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:50.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:50.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:50.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:50.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:50.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:50.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:50.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:51.008 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:51.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:51.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:51.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:51.190 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:51.190 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:51.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:51.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:51.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:51.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:51.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:51.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:51.532 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:21:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:21:51] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-10-09T06:21:51.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:51.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:51.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:51.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:51.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:51.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:51.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:51.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:51.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: cluster 2023-10-09T06:21:50.751272+0000 mgr.y (mgr.25723) 497 : cluster 0 pgmap v459: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 11074/68493 objects degraded (16.168%); 9679/68493 objects misplaced (14.131%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: audit 2023-10-09T06:21:51.193605+0000 mon.a (mon.0) 681 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: audit 2023-10-09T06:21:51.203485+0000 mon.a (mon.0) 682 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: audit 2023-10-09T06:21:51.846963+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: audit 2023-10-09T06:21:51.848258+0000 mon.a (mon.0) 684 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.059 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[77299]: audit 2023-10-09T06:21:51.854728+0000 mon.a (mon.0) 685 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: cluster 2023-10-09T06:21:50.751272+0000 mgr.y (mgr.25723) 497 : cluster 0 pgmap v459: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 11074/68493 objects degraded (16.168%); 9679/68493 objects misplaced (14.131%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: audit 2023-10-09T06:21:51.193605+0000 mon.a (mon.0) 681 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: audit 2023-10-09T06:21:51.203485+0000 mon.a (mon.0) 682 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: audit 2023-10-09T06:21:51.846963+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: audit 2023-10-09T06:21:51.848258+0000 mon.a (mon.0) 684 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.060 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:52 smithi125 bash[80002]: audit 2023-10-09T06:21:51.854728+0000 mon.a (mon.0) 685 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:52.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: cluster 2023-10-09T06:21:50.751272+0000 mgr.y (mgr.25723) 497 : cluster 0 pgmap v459: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 11074/68493 objects degraded (16.168%); 9679/68493 objects misplaced (14.131%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:52.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: audit 2023-10-09T06:21:51.193605+0000 mon.a (mon.0) 681 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: audit 2023-10-09T06:21:51.203485+0000 mon.a (mon.0) 682 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: audit 2023-10-09T06:21:51.846963+0000 mon.a (mon.0) 683 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.081 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: audit 2023-10-09T06:21:51.848258+0000 mon.a (mon.0) 684 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.081 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:52 smithi155 bash[59495]: audit 2023-10-09T06:21:51.854728+0000 mon.a (mon.0) 685 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:21:52.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:52.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:21:52.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:52.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:52.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:52.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:52.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:52.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:52.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:52.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:52.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:52.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:52.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:52.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:52.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:53.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:53.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:53.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:53.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:53.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:53.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:53.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:53.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:53.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:53.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:53.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:53.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:53.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:54.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:54.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:54.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:54 smithi155 bash[59495]: cluster 2023-10-09T06:21:52.754955+0000 mgr.y (mgr.25723) 498 : cluster 0 pgmap v460: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11074/64266 objects degraded (17.232%); 9679/64266 objects misplaced (15.061%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:54.121 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:54 smithi155 bash[59495]: cluster 2023-10-09T06:21:53.653865+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 11074/64266 objects degraded (17.232%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:54.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:54.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:54.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:54.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:54 smithi125 bash[77299]: cluster 2023-10-09T06:21:52.754955+0000 mgr.y (mgr.25723) 498 : cluster 0 pgmap v460: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11074/64266 objects degraded (17.232%); 9679/64266 objects misplaced (15.061%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:54.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:54 smithi125 bash[77299]: cluster 2023-10-09T06:21:53.653865+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 11074/64266 objects degraded (17.232%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:54.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:54 smithi125 bash[80002]: cluster 2023-10-09T06:21:52.754955+0000 mgr.y (mgr.25723) 498 : cluster 0 pgmap v460: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 11074/64266 objects degraded (17.232%); 9679/64266 objects misplaced (15.061%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:54.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:54 smithi125 bash[80002]: cluster 2023-10-09T06:21:53.653865+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 11074/64266 objects degraded (17.232%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:54.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:54.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:54.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:54.582 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:54.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:54.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:54.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:54.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:54.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:55.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:55.112 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:55.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:55.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:55.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:55.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:55.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:55.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:55.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:55.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:55.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:55.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:55.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:55.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:55.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:55.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:56.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:56.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:56.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:56.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:56.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:56.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:56.340 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:56 smithi125 bash[77299]: cluster 2023-10-09T06:21:54.755712+0000 mgr.y (mgr.25723) 499 : cluster 0 pgmap v461: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11074/62121 objects degraded (17.826%); 9679/62121 objects misplaced (15.581%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:56.340 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:56 smithi125 bash[80002]: cluster 2023-10-09T06:21:54.755712+0000 mgr.y (mgr.25723) 499 : cluster 0 pgmap v461: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11074/62121 objects degraded (17.826%); 9679/62121 objects misplaced (15.581%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:56.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:56.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:56 smithi155 bash[59495]: cluster 2023-10-09T06:21:54.755712+0000 mgr.y (mgr.25723) 499 : cluster 0 pgmap v461: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 11074/62121 objects degraded (17.826%); 9679/62121 objects misplaced (15.581%); 154 KiB/s, 21 objects/s recovering 2023-10-09T06:21:56.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:56.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:56.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:56.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:56.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:56.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:56.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:56.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:56.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:57.016 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:57.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:57.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:57.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:57.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:57.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:57.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:57.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:57.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:57.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:57.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:57.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:57.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:57.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:57.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:57.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:57.968 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:21:57.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:58.003 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:21:58.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:58.045 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:21:58.094 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:21:58.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:58.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:58 smithi155 bash[59495]: cluster 2023-10-09T06:21:56.756345+0000 mgr.y (mgr.25723) 500 : cluster 0 pgmap v462: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10966/55143 objects degraded (19.886%); 9679/55143 objects misplaced (17.553%); 0 B/s, 21 objects/s recovering 2023-10-09T06:21:58.130 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:58 smithi155 bash[59495]: audit 2023-10-09T06:21:57.926997+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.125:0/2271643194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:58.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:58.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:58.154 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:21:58.193 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:21:58.227 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:21:58.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:58.262 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:21:58.271 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.271 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.271 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.271 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.271 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.272 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.272 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.272 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:58.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:58.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:58 smithi125 bash[77299]: cluster 2023-10-09T06:21:56.756345+0000 mgr.y (mgr.25723) 500 : cluster 0 pgmap v462: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10966/55143 objects degraded (19.886%); 9679/55143 objects misplaced (17.553%); 0 B/s, 21 objects/s recovering 2023-10-09T06:21:58.309 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:58 smithi125 bash[77299]: audit 2023-10-09T06:21:57.926997+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.125:0/2271643194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:58.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:58 smithi125 bash[80002]: cluster 2023-10-09T06:21:56.756345+0000 mgr.y (mgr.25723) 500 : cluster 0 pgmap v462: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10966/55143 objects degraded (19.886%); 9679/55143 objects misplaced (17.553%); 0 B/s, 21 objects/s recovering 2023-10-09T06:21:58.309 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:58 smithi125 bash[80002]: audit 2023-10-09T06:21:57.926997+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.125:0/2271643194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:58.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:58.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:58.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:58.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:58.668 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:58.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:58.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:58.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:58.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:58.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:58.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:58.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:59.036 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:59 smithi155 bash[59495]: cluster 2023-10-09T06:21:58.655057+0000 mon.a (mon.0) 687 : cluster 3 Health check update: Degraded data redundancy: 10966/55143 objects degraded (19.886%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:59.037 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:59 smithi155 bash[59495]: audit 2023-10-09T06:21:58.769686+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.125:0/686788602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.037 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:21:59 smithi155 bash[59495]: audit 2023-10-09T06:21:58.770450+0000 mon.a (mon.0) 688 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:59.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:59.096 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[77299]: cluster 2023-10-09T06:21:58.655057+0000 mon.a (mon.0) 687 : cluster 3 Health check update: Degraded data redundancy: 10966/55143 objects degraded (19.886%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:59.096 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[77299]: audit 2023-10-09T06:21:58.769686+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.125:0/686788602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.096 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[77299]: audit 2023-10-09T06:21:58.770450+0000 mon.a (mon.0) 688 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.097 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[80002]: cluster 2023-10-09T06:21:58.655057+0000 mon.a (mon.0) 687 : cluster 3 Health check update: Degraded data redundancy: 10966/55143 objects degraded (19.886%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:21:59.097 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[80002]: audit 2023-10-09T06:21:58.769686+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.125:0/686788602' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.097 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:21:59 smithi125 bash[80002]: audit 2023-10-09T06:21:58.770450+0000 mon.a (mon.0) 688 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:59.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:59.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:59.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:59.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:59.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:59.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:59.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:59.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:59.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:59.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:59.690 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:21:59.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:59.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:59.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:59.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:21:59.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:00.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:00.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:00.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:00.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:00.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:00.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:00.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:00 smithi155 bash[59495]: cluster 2023-10-09T06:21:58.757001+0000 mgr.y (mgr.25723) 501 : cluster 0 pgmap v463: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10966/53406 objects degraded (20.533%); 9679/53406 objects misplaced (18.123%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:00 smithi155 bash[59495]: cluster 2023-10-09T06:21:59.657649+0000 mon.a (mon.0) 689 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:00 smithi155 bash[59495]: audit 2023-10-09T06:21:59.675985+0000 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:00.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:00 smithi155 bash[59495]: cluster 2023-10-09T06:21:59.676073+0000 mon.a (mon.0) 691 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:22:00.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:00.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:00.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[77299]: cluster 2023-10-09T06:21:58.757001+0000 mgr.y (mgr.25723) 501 : cluster 0 pgmap v463: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10966/53406 objects degraded (20.533%); 9679/53406 objects misplaced (18.123%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:00.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[77299]: cluster 2023-10-09T06:21:59.657649+0000 mon.a (mon.0) 689 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:00.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[77299]: audit 2023-10-09T06:21:59.675985+0000 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:00.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[77299]: cluster 2023-10-09T06:21:59.676073+0000 mon.a (mon.0) 691 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:22:00.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[80002]: cluster 2023-10-09T06:21:58.757001+0000 mgr.y (mgr.25723) 501 : cluster 0 pgmap v463: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 10966/53406 objects degraded (20.533%); 9679/53406 objects misplaced (18.123%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:00.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[80002]: cluster 2023-10-09T06:21:59.657649+0000 mon.a (mon.0) 689 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:00.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[80002]: audit 2023-10-09T06:21:59.675985+0000 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:00.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:00 smithi125 bash[80002]: cluster 2023-10-09T06:21:59.676073+0000 mon.a (mon.0) 691 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:22:00.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:00.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:00.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:00.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:00.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:00.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:00.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:00.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:00.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:00.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:01.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:01.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:01.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:01.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:01.305 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:01 smithi155 bash[59495]: cluster 2023-10-09T06:21:59.678012+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.305 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:01 smithi155 bash[59495]: cluster 2023-10-09T06:21:59.686495+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:01.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:01 smithi125 bash[77299]: cluster 2023-10-09T06:21:59.678012+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:01 smithi125 bash[77299]: cluster 2023-10-09T06:21:59.686495+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:01] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.33.4" 2023-10-09T06:22:01.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:01 smithi125 bash[80002]: cluster 2023-10-09T06:21:59.678012+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:01 smithi125 bash[80002]: cluster 2023-10-09T06:21:59.686495+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e135 with expected crc 2023-10-09T06:22:01.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.430 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:01.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:01.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:01.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:01.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:192808 5.b e135/105 5:dff6cb99:::benchmark_data_smithi125_106927_object89146:head v 135'12052, mlcod=135'12050)", 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.701330+0000", 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0090298259999999995, 2023-10-09T06:22:01.712 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090613069999999993, 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701330+0000", 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.713 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701330+0000", 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701329+0000", 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5220000000000007e-06 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.714 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701338+0000", 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1304000000000001e-05 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701349+0000", 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8589999999999999e-06 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701356+0000", 2023-10-09T06:22:01.715 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6984999999999998e-05 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701403+0000", 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7589e-05 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701420+0000", 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0365999999999995e-05 2023-10-09T06:22:01.716 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:192807 5.7 5.2e949f77 (undecoded) ondisk+write+known_if_redirected e135)", 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.701780+0000", 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0085800570000000003, 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0086832439999999997, 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.717 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192807 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701780+0000", 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.718 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701780+0000", 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701778+0000", 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0409999999999998e-06 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.719 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701786+0000", 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1348999999999999e-05 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701798+0000", 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.195e-06 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.701804+0000", 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3995e-05 2023-10-09T06:22:01.720 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:192810 5.d e135/118 5:bcd9ca37:::benchmark_data_smithi125_106927_object89148:head v 135'12081, mlcod=0'0)", 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.705711+0000", 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0046490109999999998, 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048179440000000002, 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:22:01.721 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705711+0000", 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705711+0000", 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.722 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705710+0000", 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0362e-05 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705720+0000", 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6314999999999999e-05 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.723 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705736+0000", 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5970000000000003e-06 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705744+0000", 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014782099999999999 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705892+0000", 2023-10-09T06:22:01.724 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7871000000000001e-05 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705920+0000", 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020884300000000001 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44693.0:192797 5.7 e135/105)", 2023-10-09T06:22:01.725 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.705734+0000", 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.004626219, 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048503779999999998, 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705734+0000", 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.726 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705734+0000", 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705733+0000", 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6470000000000001e-06 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.727 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705740+0000", 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0743e-05 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705750+0000", 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.003e-06 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705756+0000", 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2654999999999999e-05 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44693.0:192797 5.7 e135/105)", 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.705777+0000", 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0045825680000000004, 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048350479999999998, 2023-10-09T06:22:01.729 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705777+0000", 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.730 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705777+0000", 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705777+0000", 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4649999999999999e-06 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705783+0000", 2023-10-09T06:22:01.731 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5221999999999999e-05 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705798+0000", 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3899999999999998e-06 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705805+0000", 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7580999999999999e-05 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.732 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44693.0:192798 5.7 e135/105)", 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.705811+0000", 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0045493160000000003, 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048309260000000001, 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.733 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705811+0000", 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705811+0000", 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.734 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705810+0000", 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9660000000000001e-06 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705813+0000", 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8190000000000003e-06 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.735 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705820+0000", 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9020000000000001e-06 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.705824+0000", 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3338e-05 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.736 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop_reply(client.44693.0:192798 5.7 e135/105)", 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.709706+0000", 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00065337299999999995, 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00096185200000000002, 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709706+0000", 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.737 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709706+0000", 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709705+0000", 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5537999999999999e-05 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.738 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709720+0000", 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7371999999999999e-05 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709738+0000", 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7000000000000003e-06 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.739 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709747+0000", 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0766999999999998e-05 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:192809 5.b e135/105)", 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.709709+0000", 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00065078599999999999, 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00098637099999999999, 2023-10-09T06:22:01.740 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709709+0000", 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709709+0000", 2023-10-09T06:22:01.741 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709708+0000", 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2537999999999999e-05 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709720+0000", 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8842999999999999e-05 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.742 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709739+0000", 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6980000000000004e-06 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709748+0000", 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8967000000000001e-05 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.743 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:192813 5.e e135/105 5:7f0eaae6:::benchmark_data_smithi125_106927_object89151:head v 135'11899, mlcod=135'11897)", 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.709774+0000", 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00058568400000000005, 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000950627, 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.744 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709774+0000", 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709774+0000", 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709774+0000", 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2360000000000001e-06 2023-10-09T06:22:01.745 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709780+0000", 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3230000000000004e-06 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709789+0000", 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1350000000000002e-06 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.746 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709793+0000", 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7669000000000001e-05 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709831+0000", 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3550999999999999e-05 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.709854+0000", 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0315000000000006e-05 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:192797 5.7 5:ef6a625d:::benchmark_data_smithi125_106927_object89135:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.680500+0000", 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.029860050999999999, 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030265429999999999, 2023-10-09T06:22:01.748 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192797 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680500+0000", 2023-10-09T06:22:01.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680500+0000", 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680498+0000", 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0352e-05 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680509+0000", 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1174000000000001e-05 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680530+0000", 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5359999999999998e-06 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680536+0000", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082706619999999998 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.688807+0000", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000185417 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.688992+0000", 2023-10-09T06:22:01.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1969e-05 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.689084+0000", 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011528969 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.700613+0000", 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020113509000000002 2023-10-09T06:22:01.753 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:192798 5.7 5:e98c38fe:::benchmark_data_smithi125_106927_object89136:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:01.680547+0000", 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.029812972, 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030267167000000001, 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:22:01.754 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 192798 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680547+0000", 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680547+0000", 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680546+0000", 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1640000000000001e-06 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680550+0000", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2939999999999999e-06 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680554+0000", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7879999999999998e-06 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.680557+0000", 2023-10-09T06:22:01.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014296608000000001 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.694853+0000", 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105505 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.694959+0000", 2023-10-09T06:22:01.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000124993 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.695084+0000", 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055337930000000004 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:01.700618+0000", 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020070660000000001 2023-10-09T06:22:01.759 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 11 2023-10-09T06:22:01.760 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:22:01.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:01.934 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:01.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:01.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:02.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:02.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:02 smithi155 bash[59495]: cluster 2023-10-09T06:22:00.757566+0000 mgr.y (mgr.25723) 502 : cluster 0 pgmap v465: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10861/50661 objects degraded (21.439%); 9679/50661 objects misplaced (19.105%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:02.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:02.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:02.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:02.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:02.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:02.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:02 smithi125 bash[77299]: cluster 2023-10-09T06:22:00.757566+0000 mgr.y (mgr.25723) 502 : cluster 0 pgmap v465: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10861/50661 objects degraded (21.439%); 9679/50661 objects misplaced (19.105%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:02.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:02 smithi125 bash[80002]: cluster 2023-10-09T06:22:00.757566+0000 mgr.y (mgr.25723) 502 : cluster 0 pgmap v465: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 10861/50661 objects degraded (21.439%); 9679/50661 objects misplaced (19.105%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:02.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:02.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:02.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:02.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:02.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:02.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:22:02.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:22:02.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:02.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:02.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:02.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:02.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:03.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:03.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:03.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:03.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:03.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:03.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:03.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:03.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:03.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:03.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:03.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:03.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:03.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:03.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:03.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:04.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: cluster 2023-10-09T06:22:02.758242+0000 mgr.y (mgr.25723) 503 : cluster 0 pgmap v466: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 965 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 10861/46167 objects degraded (23.525%); 9679/46167 objects misplaced (20.965%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:04.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:04.087 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:22:04.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:04.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:04.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:04.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: cluster 2023-10-09T06:22:02.758242+0000 mgr.y (mgr.25723) 503 : cluster 0 pgmap v466: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 965 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 10861/46167 objects degraded (23.525%); 9679/46167 objects misplaced (20.965%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:04.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: audit 2023-10-09T06:22:03.091117+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.125:0/2598108330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: audit 2023-10-09T06:22:03.091528+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: audit 2023-10-09T06:22:03.582534+0000 mon.a (mon.0) 693 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: audit 2023-10-09T06:22:03.583570+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[77299]: cluster 2023-10-09T06:22:03.656211+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 10861/46167 objects degraded (23.525%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: cluster 2023-10-09T06:22:02.758242+0000 mgr.y (mgr.25723) 503 : cluster 0 pgmap v466: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 965 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 10861/46167 objects degraded (23.525%); 9679/46167 objects misplaced (20.965%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: audit 2023-10-09T06:22:03.091117+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.125:0/2598108330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: audit 2023-10-09T06:22:03.091528+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: audit 2023-10-09T06:22:03.582534+0000 mon.a (mon.0) 693 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:04.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: audit 2023-10-09T06:22:03.583570+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:04.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:04 smithi125 bash[80002]: cluster 2023-10-09T06:22:03.656211+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 10861/46167 objects degraded (23.525%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:04.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:22:04.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:04.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:04.337 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: audit 2023-10-09T06:22:03.091117+0000 mon.c (mon.2) 51 : audit 1 from='client.? 172.21.15.125:0/2598108330' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.337 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: audit 2023-10-09T06:22:03.091528+0000 mon.a (mon.0) 692 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.337 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: audit 2023-10-09T06:22:03.582534+0000 mon.a (mon.0) 693 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:04.337 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: audit 2023-10-09T06:22:03.583570+0000 mon.a (mon.0) 694 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:04.338 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:04 smithi155 bash[59495]: cluster 2023-10-09T06:22:03.656211+0000 mon.a (mon.0) 695 : cluster 3 Health check update: Degraded data redundancy: 10861/46167 objects degraded (23.525%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:04.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:04.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:04.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:04.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:04.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:04.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:04.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:05.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:05.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:05.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:05.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:05.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:05.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[77299]: audit 2023-10-09T06:22:04.068751+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:05.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[77299]: cluster 2023-10-09T06:22:04.068917+0000 mon.a (mon.0) 697 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:22:05.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[77299]: cluster 2023-10-09T06:22:04.071184+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[77299]: cluster 2023-10-09T06:22:04.071285+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[80002]: audit 2023-10-09T06:22:04.068751+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:05.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[80002]: cluster 2023-10-09T06:22:04.068917+0000 mon.a (mon.0) 697 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:22:05.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[80002]: cluster 2023-10-09T06:22:04.071184+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.328 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:05 smithi125 bash[80002]: cluster 2023-10-09T06:22:04.071285+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:05.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:05 smithi155 bash[59495]: audit 2023-10-09T06:22:04.068751+0000 mon.a (mon.0) 696 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:05.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:05 smithi155 bash[59495]: cluster 2023-10-09T06:22:04.068917+0000 mon.a (mon.0) 697 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:22:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:05 smithi155 bash[59495]: cluster 2023-10-09T06:22:04.071184+0000 osd.4 (osd.4) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:05 smithi155 bash[59495]: cluster 2023-10-09T06:22:04.071285+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e136 with expected crc 2023-10-09T06:22:05.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:22:05.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:05.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:05.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:05.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:05.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:05.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:05.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:05.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:05.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:06.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:06.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:06 smithi155 bash[59495]: cluster 2023-10-09T06:22:04.758914+0000 mgr.y (mgr.25723) 504 : cluster 0 pgmap v468: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 10861/43851 objects degraded (24.768%); 9679/43851 objects misplaced (22.072%); 0 B/s, 13 objects/s recovering 2023-10-09T06:22:06.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:06.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:06.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:06 smithi125 bash[77299]: cluster 2023-10-09T06:22:04.758914+0000 mgr.y (mgr.25723) 504 : cluster 0 pgmap v468: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 10861/43851 objects degraded (24.768%); 9679/43851 objects misplaced (22.072%); 0 B/s, 13 objects/s recovering 2023-10-09T06:22:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:06 smithi125 bash[80002]: cluster 2023-10-09T06:22:04.758914+0000 mgr.y (mgr.25723) 504 : cluster 0 pgmap v468: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 916 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 470 op/s; 10861/43851 objects degraded (24.768%); 9679/43851 objects misplaced (22.072%); 0 B/s, 13 objects/s recovering 2023-10-09T06:22:06.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:06.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:06.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:06.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:06.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:06.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:06.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:06.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:06.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:06.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:07.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:07.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:07.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:07.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:07.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:07.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:07.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:07.532 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:07.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:07.606 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:07.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:07.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:07.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:07.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:07.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:07.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:07.904 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:07.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:08.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:08.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:08.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:08 smithi125 bash[77299]: cluster 2023-10-09T06:22:06.759718+0000 mgr.y (mgr.25723) 505 : cluster 0 pgmap v469: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 778 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10756/37188 objects degraded (28.923%); 9679/37188 objects misplaced (26.027%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:08.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:08 smithi125 bash[80002]: cluster 2023-10-09T06:22:06.759718+0000 mgr.y (mgr.25723) 505 : cluster 0 pgmap v469: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 778 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10756/37188 objects degraded (28.923%); 9679/37188 objects misplaced (26.027%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:08.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:08.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:08.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:08 smithi155 bash[59495]: cluster 2023-10-09T06:22:06.759718+0000 mgr.y (mgr.25723) 505 : cluster 0 pgmap v469: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 778 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10756/37188 objects degraded (28.923%); 9679/37188 objects misplaced (26.027%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:08.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:08.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:08.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:08.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:08.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:08.812 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:08.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:08.922 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:08.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:08.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:09.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:09.090 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:09 smithi155 bash[59495]: cluster 2023-10-09T06:22:08.657078+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 10756/37188 objects degraded (28.923%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:09.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:09.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:09.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:09.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:09.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:09.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:09.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:09.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:09 smithi125 bash[77299]: cluster 2023-10-09T06:22:08.657078+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 10756/37188 objects degraded (28.923%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:09.492 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:09 smithi125 bash[80002]: cluster 2023-10-09T06:22:08.657078+0000 mon.a (mon.0) 698 : cluster 3 Health check update: Degraded data redundancy: 10756/37188 objects degraded (28.923%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:09.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:09.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:09.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:09.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:09.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:09.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:09.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:09.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:09.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:09.943 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:10.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:10.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:10.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:10 smithi125 bash[77299]: cluster 2023-10-09T06:22:08.760253+0000 mgr.y (mgr.25723) 506 : cluster 0 pgmap v470: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 742 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10756/35484 objects degraded (30.312%); 9679/35484 objects misplaced (27.277%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:10.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:10.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:10.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:10.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:10.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:10 smithi125 bash[80002]: cluster 2023-10-09T06:22:08.760253+0000 mgr.y (mgr.25723) 506 : cluster 0 pgmap v470: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 742 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10756/35484 objects degraded (30.312%); 9679/35484 objects misplaced (27.277%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:10.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:10 smithi155 bash[59495]: cluster 2023-10-09T06:22:08.760253+0000 mgr.y (mgr.25723) 506 : cluster 0 pgmap v470: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 742 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 10756/35484 objects degraded (30.312%); 9679/35484 objects misplaced (27.277%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:10.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:10.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:10.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:10.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:10.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:10.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:10.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:10.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:10.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:10.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:10.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:10.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:10.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:11.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:11.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:11.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:11.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:11.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:11.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:11.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:11.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:11] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.33.4" 2023-10-09T06:22:11.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:11.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:11.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:11.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:11.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:12.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:12.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:12.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:12.198 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:12 smithi155 bash[59495]: cluster 2023-10-09T06:22:10.761021+0000 mgr.y (mgr.25723) 507 : cluster 0 pgmap v471: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 684 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10650/32694 objects degraded (32.575%); 9679/32694 objects misplaced (29.605%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:12.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:12.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:12.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:12 smithi125 bash[77299]: cluster 2023-10-09T06:22:10.761021+0000 mgr.y (mgr.25723) 507 : cluster 0 pgmap v471: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 684 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10650/32694 objects degraded (32.575%); 9679/32694 objects misplaced (29.605%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:12.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:12 smithi125 bash[80002]: cluster 2023-10-09T06:22:10.761021+0000 mgr.y (mgr.25723) 507 : cluster 0 pgmap v471: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 684 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 10650/32694 objects degraded (32.575%); 9679/32694 objects misplaced (29.605%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:12.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:12.416 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:22:12.416 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:22:12.416 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:22:12.417 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:22:12.417 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:22:12.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:12.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:12.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:12.603 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:12.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:12.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:12.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:12.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:12.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:12.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:13.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:13.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:13.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:13.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:13.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:13.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:13.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:13.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:13.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:22:13.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:13.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:13.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:13.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:13.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:13.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:13.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:13.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:13.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:13.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:14.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:14.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:14.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:14.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:14 smithi125 bash[77299]: cluster 2023-10-09T06:22:12.761716+0000 mgr.y (mgr.25723) 508 : cluster 0 pgmap v472: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 586 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10650/28011 objects degraded (38.021%); 9679/28011 objects misplaced (34.554%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:14.250 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:14 smithi125 bash[77299]: cluster 2023-10-09T06:22:13.658248+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 10650/28011 objects degraded (38.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:14.251 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:14 smithi125 bash[80002]: cluster 2023-10-09T06:22:12.761716+0000 mgr.y (mgr.25723) 508 : cluster 0 pgmap v472: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 586 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10650/28011 objects degraded (38.021%); 9679/28011 objects misplaced (34.554%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:14.251 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:14 smithi125 bash[80002]: cluster 2023-10-09T06:22:13.658248+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 10650/28011 objects degraded (38.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:14.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:14.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:14.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:14 smithi155 bash[59495]: cluster 2023-10-09T06:22:12.761716+0000 mgr.y (mgr.25723) 508 : cluster 0 pgmap v472: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 586 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10650/28011 objects degraded (38.021%); 9679/28011 objects misplaced (34.554%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:14.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:14 smithi155 bash[59495]: cluster 2023-10-09T06:22:13.658248+0000 mon.a (mon.0) 699 : cluster 3 Health check update: Degraded data redundancy: 10650/28011 objects degraded (38.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:14.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:14.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:14.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:14.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:14.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:14.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:14.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:14.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:14.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:14.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:14.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:14.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:15.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:15.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:15.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:15.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:15.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:15.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:15.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:15.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:15.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:15.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:15.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:15.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:15.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:16.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:16.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:16 smithi125 bash[80002]: cluster 2023-10-09T06:22:14.762360+0000 mgr.y (mgr.25723) 509 : cluster 0 pgmap v473: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 537 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 10650/25623 objects degraded (41.564%); 9679/25623 objects misplaced (37.775%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:16.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:16.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:16.303 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:16 smithi155 bash[59495]: cluster 2023-10-09T06:22:14.762360+0000 mgr.y (mgr.25723) 509 : cluster 0 pgmap v473: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 537 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 10650/25623 objects degraded (41.564%); 9679/25623 objects misplaced (37.775%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:16.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:16.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:16 smithi125 bash[77299]: cluster 2023-10-09T06:22:14.762360+0000 mgr.y (mgr.25723) 509 : cluster 0 pgmap v473: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 537 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 10650/25623 objects degraded (41.564%); 9679/25623 objects misplaced (37.775%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:16.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:16.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:16.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:16.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:16.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:16.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:16.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:16.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:16.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:16.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:17.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:17.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:17.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:17.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:17.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:17.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:17.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:17.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:17.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:17.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:17.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:17.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:17.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:17.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:17.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:17.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:17.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:18.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:18 smithi125 bash[77299]: cluster 2023-10-09T06:22:16.763036+0000 mgr.y (mgr.25723) 510 : cluster 0 pgmap v474: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 402 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 10545/19161 objects degraded (55.034%); 9679/19161 objects misplaced (50.514%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:18.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:18.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.262 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:18 smithi155 bash[59495]: cluster 2023-10-09T06:22:16.763036+0000 mgr.y (mgr.25723) 510 : cluster 0 pgmap v474: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 402 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 10545/19161 objects degraded (55.034%); 9679/19161 objects misplaced (50.514%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:18.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:18.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:18 smithi125 bash[80002]: cluster 2023-10-09T06:22:16.763036+0000 mgr.y (mgr.25723) 510 : cluster 0 pgmap v474: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 402 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 10545/19161 objects degraded (55.034%); 9679/19161 objects misplaced (50.514%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:18.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:18.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:18.442 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:18.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.583 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:18.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:18.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:18.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:18.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:19.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:19.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:19 smithi155 bash[59495]: audit 2023-10-09T06:22:18.569628+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:19.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:19 smithi155 bash[59495]: cluster 2023-10-09T06:22:18.659332+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 10545/19161 objects degraded (55.034%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:19.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:19.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:19.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:19.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:19 smithi125 bash[77299]: audit 2023-10-09T06:22:18.569628+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:19.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:19 smithi125 bash[77299]: cluster 2023-10-09T06:22:18.659332+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 10545/19161 objects degraded (55.034%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:19.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:19 smithi125 bash[80002]: audit 2023-10-09T06:22:18.569628+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:19.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:19 smithi125 bash[80002]: cluster 2023-10-09T06:22:18.659332+0000 mon.a (mon.0) 701 : cluster 3 Health check update: Degraded data redundancy: 10545/19161 objects degraded (55.034%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:19.397 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:22:19.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:19.441 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:22:19.481 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:22:19.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:19.519 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:22:19.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:19.554 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:22:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:19.588 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:22:19.625 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:22:19.658 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:22:19.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.664 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.666 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:22:19.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:19.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:19.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:19.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:19.984 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:19.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:20.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:20.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:20.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:20.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:20.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:20.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:20 smithi125 bash[77299]: cluster 2023-10-09T06:22:18.763507+0000 mgr.y (mgr.25723) 511 : cluster 0 pgmap v475: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 367 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 10545/17490 objects degraded (60.292%); 9679/17490 objects misplaced (55.340%); 0 B/s, 17 objects/s recovering 2023-10-09T06:22:20.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:20 smithi125 bash[77299]: audit 2023-10-09T06:22:19.342865+0000 mon.a (mon.0) 702 : audit 0 from='client.? 172.21.15.125:0/3339131604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:20.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:20 smithi125 bash[80002]: cluster 2023-10-09T06:22:18.763507+0000 mgr.y (mgr.25723) 511 : cluster 0 pgmap v475: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 367 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 10545/17490 objects degraded (60.292%); 9679/17490 objects misplaced (55.340%); 0 B/s, 17 objects/s recovering 2023-10-09T06:22:20.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:20 smithi125 bash[80002]: audit 2023-10-09T06:22:19.342865+0000 mon.a (mon.0) 702 : audit 0 from='client.? 172.21.15.125:0/3339131604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:20.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:20 smithi155 bash[59495]: cluster 2023-10-09T06:22:18.763507+0000 mgr.y (mgr.25723) 511 : cluster 0 pgmap v475: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 367 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 10545/17490 objects degraded (60.292%); 9679/17490 objects misplaced (55.340%); 0 B/s, 17 objects/s recovering 2023-10-09T06:22:20.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:20 smithi155 bash[59495]: audit 2023-10-09T06:22:19.342865+0000 mon.a (mon.0) 702 : audit 0 from='client.? 172.21.15.125:0/3339131604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:20.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:20.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:20.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:20.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:20.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:20.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:20.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:20.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:20.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:20.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:21.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:21.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:21.137 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:22:21.137 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:22:21.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:21 smithi155 bash[59495]: audit 2023-10-09T06:22:20.418845+0000 mon.a (mon.0) 703 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:21 smithi155 bash[59495]: audit 2023-10-09T06:22:20.423425+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.125:0/3317334010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:21 smithi155 bash[59495]: audit 2023-10-09T06:22:20.638957+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:21 smithi155 bash[59495]: audit 2023-10-09T06:22:20.642602+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.125:0/1192664641' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:21.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:21.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:21.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:21.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[77299]: audit 2023-10-09T06:22:20.418845+0000 mon.a (mon.0) 703 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[77299]: audit 2023-10-09T06:22:20.423425+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.125:0/3317334010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[77299]: audit 2023-10-09T06:22:20.638957+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[77299]: audit 2023-10-09T06:22:20.642602+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.125:0/1192664641' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:21] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-10-09T06:22:21.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[80002]: audit 2023-10-09T06:22:20.418845+0000 mon.a (mon.0) 703 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[80002]: audit 2023-10-09T06:22:20.423425+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.125:0/3317334010' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:21.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[80002]: audit 2023-10-09T06:22:20.638957+0000 mon.a (mon.0) 704 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:21 smithi125 bash[80002]: audit 2023-10-09T06:22:20.642602+0000 mon.b (mon.1) 6 : audit 1 from='client.? 172.21.15.125:0/1192664641' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:21.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:21.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:21.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:21.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:21.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:21.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:21.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:21.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:21.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:21.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:21.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:22.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:22.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:22.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: cluster 2023-10-09T06:22:20.764163+0000 mgr.y (mgr.25723) 512 : cluster 0 pgmap v476: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 309 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 10438/14691 objects degraded (71.050%); 9679/14691 objects misplaced (65.884%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: cluster 2023-10-09T06:22:21.110762+0000 mon.a (mon.0) 705 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: audit 2023-10-09T06:22:21.123116+0000 mon.a (mon.0) 706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: audit 2023-10-09T06:22:21.123236+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: cluster 2023-10-09T06:22:21.123374+0000 mon.a (mon.0) 708 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: cluster 2023-10-09T06:22:21.124353+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[77299]: cluster 2023-10-09T06:22:21.129605+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: cluster 2023-10-09T06:22:20.764163+0000 mgr.y (mgr.25723) 512 : cluster 0 pgmap v476: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 309 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 10438/14691 objects degraded (71.050%); 9679/14691 objects misplaced (65.884%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: cluster 2023-10-09T06:22:21.110762+0000 mon.a (mon.0) 705 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: audit 2023-10-09T06:22:21.123116+0000 mon.a (mon.0) 706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: audit 2023-10-09T06:22:21.123236+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: cluster 2023-10-09T06:22:21.123374+0000 mon.a (mon.0) 708 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: cluster 2023-10-09T06:22:21.124353+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:22 smithi125 bash[80002]: cluster 2023-10-09T06:22:21.129605+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:22.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: cluster 2023-10-09T06:22:20.764163+0000 mgr.y (mgr.25723) 512 : cluster 0 pgmap v476: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 309 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 10438/14691 objects degraded (71.050%); 9679/14691 objects misplaced (65.884%); 0 B/s, 26 objects/s recovering 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: cluster 2023-10-09T06:22:21.110762+0000 mon.a (mon.0) 705 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: audit 2023-10-09T06:22:21.123116+0000 mon.a (mon.0) 706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: audit 2023-10-09T06:22:21.123236+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: cluster 2023-10-09T06:22:21.123374+0000 mon.a (mon.0) 708 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:22:22.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: cluster 2023-10-09T06:22:21.124353+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:22 smithi155 bash[59495]: cluster 2023-10-09T06:22:21.129605+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e137 with expected crc 2023-10-09T06:22:22.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:22.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:22.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:22.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:22.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:22.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:22.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:22.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:22.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:22.943 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:22:22.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:22:22.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:23.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:23.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.132 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:23 smithi155 bash[59495]: cluster 2023-10-09T06:22:22.764806+0000 mgr.y (mgr.25723) 513 : cluster 0 pgmap v478: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10438/10221 objects degraded (102.123%); 9679/10221 objects misplaced (94.697%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:23.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:22:23.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.298 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:23.298 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:23.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:23.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:23 smithi125 bash[77299]: cluster 2023-10-09T06:22:22.764806+0000 mgr.y (mgr.25723) 513 : cluster 0 pgmap v478: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10438/10221 objects degraded (102.123%); 9679/10221 objects misplaced (94.697%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:23.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:23 smithi125 bash[80002]: cluster 2023-10-09T06:22:22.764806+0000 mgr.y (mgr.25723) 513 : cluster 0 pgmap v478: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 216 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10438/10221 objects degraded (102.123%); 9679/10221 objects misplaced (94.697%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:23.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:23.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:23.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:23.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:23.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:23.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:23.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:23.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:23.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:23.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:24.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:24.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:24.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:24.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:24.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:22:24.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:24.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:24 smithi125 bash[77299]: cluster 2023-10-09T06:22:23.660347+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 10438/10221 objects degraded (102.123%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:24.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:24 smithi125 bash[80002]: cluster 2023-10-09T06:22:23.660347+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 10438/10221 objects degraded (102.123%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:24 smithi155 bash[59495]: cluster 2023-10-09T06:22:23.660347+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 10438/10221 objects degraded (102.123%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:24.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:24.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:24.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:24.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:24.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:24.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:24.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:24.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:24.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:24.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.984 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:24.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:24.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:22:25.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:25.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:25.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:25.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:25.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:25.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:25 smithi125 bash[77299]: cluster 2023-10-09T06:22:24.765594+0000 mgr.y (mgr.25723) 514 : cluster 0 pgmap v479: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 10438/7947 objects degraded (131.345%); 9679/7947 objects misplaced (121.794%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:25.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:25 smithi125 bash[80002]: cluster 2023-10-09T06:22:24.765594+0000 mgr.y (mgr.25723) 514 : cluster 0 pgmap v479: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 10438/7947 objects degraded (131.345%); 9679/7947 objects misplaced (121.794%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:25.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:25.431 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:25 smithi155 bash[59495]: cluster 2023-10-09T06:22:24.765594+0000 mgr.y (mgr.25723) 514 : cluster 0 pgmap v479: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 10438/7947 objects degraded (131.345%); 9679/7947 objects misplaced (121.794%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:25.532 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:25.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:25.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:25.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:25.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:25.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:25.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:25.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:26.029 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 103659 objects 2023-10-09T06:22:26.029 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :166.83 2023-10-09T06:22:26.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:26.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:26.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:26.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:26.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:26.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:26.496 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:26.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:26.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:26.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:26.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:26.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:26.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:26.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:26.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:26.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:26.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:26.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:27.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:27.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:27.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:27.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:27.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:27.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:27.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:27.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:27.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:27.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:27.603 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:27.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:27.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:27.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:27.773 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.774 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:27.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:27.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:27.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:27.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:27 smithi155 bash[59495]: cluster 2023-10-09T06:22:26.766886+0000 mgr.y (mgr.25723) 515 : cluster 0 pgmap v480: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10335/1689 objects degraded (611.901%); 9679/1689 objects misplaced (573.061%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:27.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:27.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:28.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:28.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:28.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:27 smithi125 bash[77299]: cluster 2023-10-09T06:22:26.766886+0000 mgr.y (mgr.25723) 515 : cluster 0 pgmap v480: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10335/1689 objects degraded (611.901%); 9679/1689 objects misplaced (573.061%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:28.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:27 smithi125 bash[80002]: cluster 2023-10-09T06:22:26.766886+0000 mgr.y (mgr.25723) 515 : cluster 0 pgmap v480: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10335/1689 objects degraded (611.901%); 9679/1689 objects misplaced (573.061%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:28.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:28.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:28.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:28.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:28.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:28.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:28.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:28.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:28.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:28.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:28.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:28 smithi155 bash[59495]: cluster 2023-10-09T06:22:28.661678+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 10335/1689 objects degraded (611.901%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:28.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:28.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:29.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:29.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:29.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:28 smithi125 bash[77299]: cluster 2023-10-09T06:22:28.661678+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 10335/1689 objects degraded (611.901%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:29.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:28 smithi125 bash[80002]: cluster 2023-10-09T06:22:28.661678+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 10335/1689 objects degraded (611.901%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:29.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:29.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:29.204 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:29.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:29.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:29.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:29.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:29.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:29.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:29.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:29.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:29.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:29.805 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:29.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:29.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:29.941 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:29 smithi155 bash[59495]: cluster 2023-10-09T06:22:28.767870+0000 mgr.y (mgr.25723) 516 : cluster 0 pgmap v481: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10335/687 objects degraded (1504.367%); 9679/687 objects misplaced (1408.879%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:29.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:29.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:30.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:30.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:30.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:30.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:29 smithi125 bash[77299]: cluster 2023-10-09T06:22:28.767870+0000 mgr.y (mgr.25723) 516 : cluster 0 pgmap v481: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10335/687 objects degraded (1504.367%); 9679/687 objects misplaced (1408.879%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:30.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:29 smithi125 bash[80002]: cluster 2023-10-09T06:22:28.767870+0000 mgr.y (mgr.25723) 516 : cluster 0 pgmap v481: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 17 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10335/687 objects degraded (1504.367%); 9679/687 objects misplaced (1408.879%); 0 B/s, 20 objects/s recovering 2023-10-09T06:22:30.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:30.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:30.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:30.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:30.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:30.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:30.562 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T06:22:30.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T06:22:30.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:30.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:30.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:30.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:30.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:30.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:30.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:30.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:30.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:31.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:31.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:31.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:31.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.354 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:31] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-10-09T06:22:31.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:31.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:31.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:31.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:31.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:31.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:31.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:31.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:31.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:32.013 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:32.015 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:31 smithi155 bash[59495]: cluster 2023-10-09T06:22:30.768539+0000 mgr.y (mgr.25723) 517 : cluster 0 pgmap v482: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 10220/102 objects degraded (10019.608%); 9679/102 objects misplaced (9489.216%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:32.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:31 smithi125 bash[77299]: cluster 2023-10-09T06:22:30.768539+0000 mgr.y (mgr.25723) 517 : cluster 0 pgmap v482: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 10220/102 objects degraded (10019.608%); 9679/102 objects misplaced (9489.216%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:32.144 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:31 smithi125 bash[80002]: cluster 2023-10-09T06:22:30.768539+0000 mgr.y (mgr.25723) 517 : cluster 0 pgmap v482: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 10220/102 objects degraded (10019.608%); 9679/102 objects misplaced (9489.216%); 0 B/s, 21 objects/s recovering 2023-10-09T06:22:32.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:32.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:32.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:32.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:32.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:32.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:32.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:32.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:33.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:33.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:33.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:33.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:33.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:33.244 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:33.245 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:33.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:33.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:33.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:33.407 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:33.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:33.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:33.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:33.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:33.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:33.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:33.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:33.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:33.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:34.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:34.056 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:33 smithi155 bash[59495]: cluster 2023-10-09T06:22:32.769385+0000 mgr.y (mgr.25723) 518 : cluster 0 pgmap v483: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:34.056 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:33 smithi155 bash[59495]: audit 2023-10-09T06:22:33.574270+0000 mon.a (mon.0) 711 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:34.056 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:33 smithi155 bash[59495]: audit 2023-10-09T06:22:33.575450+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:34.056 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:33 smithi155 bash[59495]: cluster 2023-10-09T06:22:33.663359+0000 mon.a (mon.0) 713 : cluster 3 Health check update: Degraded data redundancy: 10220/84 objects degraded (12166.667%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:34.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[77299]: cluster 2023-10-09T06:22:32.769385+0000 mgr.y (mgr.25723) 518 : cluster 0 pgmap v483: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[77299]: audit 2023-10-09T06:22:33.574270+0000 mon.a (mon.0) 711 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[77299]: audit 2023-10-09T06:22:33.575450+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[77299]: cluster 2023-10-09T06:22:33.663359+0000 mon.a (mon.0) 713 : cluster 3 Health check update: Degraded data redundancy: 10220/84 objects degraded (12166.667%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[80002]: cluster 2023-10-09T06:22:32.769385+0000 mgr.y (mgr.25723) 518 : cluster 0 pgmap v483: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[80002]: audit 2023-10-09T06:22:33.574270+0000 mon.a (mon.0) 711 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[80002]: audit 2023-10-09T06:22:33.575450+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:33 smithi125 bash[80002]: cluster 2023-10-09T06:22:33.663359+0000 mon.a (mon.0) 713 : cluster 3 Health check update: Degraded data redundancy: 10220/84 objects degraded (12166.667%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:34.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:22:34.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:34.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:34.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:34.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:34.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:34.457 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:34.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:34.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:34.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:34.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:34.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:34.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:34.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:34.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:34.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:34.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:34.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:35.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:35.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:35.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:35.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:35.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:35.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:35.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:35.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:35.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:35.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:35.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:35 smithi125 bash[77299]: cluster 2023-10-09T06:22:34.769948+0000 mgr.y (mgr.25723) 519 : cluster 0 pgmap v484: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:35.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:35 smithi125 bash[80002]: cluster 2023-10-09T06:22:34.769948+0000 mgr.y (mgr.25723) 519 : cluster 0 pgmap v484: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:35.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:35.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:36.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:36.029 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:36.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:35 smithi155 bash[59495]: cluster 2023-10-09T06:22:34.769948+0000 mgr.y (mgr.25723) 519 : cluster 0 pgmap v484: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 10220/84 objects degraded (12166.667%); 9679/84 objects misplaced (11522.619%); 0 B/s, 18 objects/s recovering 2023-10-09T06:22:36.155 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:36.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:36.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:36.333 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:36.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:36.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:36.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:36.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:36.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:36.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:36.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:36.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:36.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:36.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:36.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:37.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:37.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:37.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:37.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:37.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:37.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:37.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:37.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:37.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:37.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:37.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:37.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:37.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:37.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:37.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:37.918 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:37 smithi125 bash[77299]: cluster 2023-10-09T06:22:36.770641+0000 mgr.y (mgr.25723) 520 : cluster 0 pgmap v485: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 27 objects/s recovering 2023-10-09T06:22:37.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:37 smithi125 bash[80002]: cluster 2023-10-09T06:22:36.770641+0000 mgr.y (mgr.25723) 520 : cluster 0 pgmap v485: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 27 objects/s recovering 2023-10-09T06:22:37.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:37.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:37.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:38.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:38.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:37 smithi155 bash[59495]: cluster 2023-10-09T06:22:36.770641+0000 mgr.y (mgr.25723) 520 : cluster 0 pgmap v485: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 27 objects/s recovering 2023-10-09T06:22:38.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:38.279 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:22:38.304 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:22:38.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.326 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:22:38.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:38.347 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:22:38.369 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:22:38.371 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:22:38.371 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:22:38.390 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:22:38.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:22:38.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.434 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.435 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:38.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:38.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:38.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:38.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:38.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:38.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:38.922 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:38.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:38.970 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:38 smithi125 bash[80002]: audit 2023-10-09T06:22:38.246869+0000 mon.a (mon.0) 714 : audit 0 from='client.? 172.21.15.125:0/2970177329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:38.971 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:38 smithi125 bash[80002]: cluster 2023-10-09T06:22:38.664968+0000 mon.a (mon.0) 715 : cluster 3 Health check update: Degraded data redundancy: 10104/84 objects degraded (12028.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:38.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:38 smithi125 bash[77299]: audit 2023-10-09T06:22:38.246869+0000 mon.a (mon.0) 714 : audit 0 from='client.? 172.21.15.125:0/2970177329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:38.971 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:38 smithi125 bash[77299]: cluster 2023-10-09T06:22:38.664968+0000 mon.a (mon.0) 715 : cluster 3 Health check update: Degraded data redundancy: 10104/84 objects degraded (12028.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:38.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:38.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:38.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:39.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:39.106 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:38 smithi155 bash[59495]: audit 2023-10-09T06:22:38.246869+0000 mon.a (mon.0) 714 : audit 0 from='client.? 172.21.15.125:0/2970177329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:39.106 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:38 smithi155 bash[59495]: cluster 2023-10-09T06:22:38.664968+0000 mon.a (mon.0) 715 : cluster 3 Health check update: Degraded data redundancy: 10104/84 objects degraded (12028.571%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:39.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:39.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.249 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:39.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:39.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:39.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:39.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:39.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:39.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:39.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:39.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:39.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:39 smithi155 bash[59495]: audit 2023-10-09T06:22:38.349003+0000 mgr.y (mgr.25723) 521 : audit 0 from='client.44861 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:39.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:39 smithi155 bash[59495]: cluster 2023-10-09T06:22:38.771653+0000 mgr.y (mgr.25723) 522 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:39.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:39.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:39.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.999 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:39 smithi125 bash[80002]: audit 2023-10-09T06:22:38.349003+0000 mgr.y (mgr.25723) 521 : audit 0 from='client.44861 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:39.999 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:39 smithi125 bash[80002]: cluster 2023-10-09T06:22:38.771653+0000 mgr.y (mgr.25723) 522 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:40.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:39 smithi125 bash[77299]: audit 2023-10-09T06:22:38.349003+0000 mgr.y (mgr.25723) 521 : audit 0 from='client.44861 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:40.000 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:39 smithi125 bash[77299]: cluster 2023-10-09T06:22:38.771653+0000 mgr.y (mgr.25723) 522 : cluster 0 pgmap v486: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 10104/84 objects degraded (12028.571%); 9679/84 objects misplaced (11522.619%); 0 B/s, 19 objects/s recovering 2023-10-09T06:22:40.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:40.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:40.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:40.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:40.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:40.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:40.381 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":485,"stamp":"2023-10-09T06:22:36.770113+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10104,"num_objects_misplaced":9679,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6290,"num_read_kb":13749,"num_write":208426,"num_write_kb":6649336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4246,"num_bytes_recovered":197208596,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144290,"ondisk_log_size":144290,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2630032,"kb_used_data":936528,"kb_used_omap":30,"kb_used_meta":1693409,"kb_avail":747331184,"statfs":{"total":767960285184,"available":765267132416,"internally_reserved":0,"allocated":959004672,"data_stored":954504742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1734050927},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-171704344,"num_objects":-2621,"num_object_clones":0,"num_object_copies":-7863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-334,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2621,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":334,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005104"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.089044+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:22:04.089044+0000","last_peered":"2023-10-09T06:22:04.089044+0000","last_clean":"2023-10-09T06:22:04.089044+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:22:04.089044+0000","last_undegraded":"2023-10-09T06:22:04.089044+0000","last_fullsized":"2023-10-09T06:22:04.089044+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:10.794972+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:22:10.794972+0000","last_peered":"2023-10-09T06:22:10.794972+0000","last_clean":"2023-10-09T06:22:10.794972+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:22:10.794972+0000","last_undegraded":"2023-10-09T06:22:10.794972+0000","last_fullsized":"2023-10-09T06:22:10.794972+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.092701+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:22:04.092701+0000","last_peered":"2023-10-09T06:22:04.092701+0000","last_clean":"2023-10-09T06:22:04.092701+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:22:04.092701+0000","last_undegraded":"2023-10-09T06:22:04.092701+0000","last_fullsized":"2023-10-09T06:22:04.092701+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077059+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:22:04.077059+0000","last_peered":"2023-10-09T06:22:04.077059+0000","last_clean":"2023-10-09T06:22:04.077059+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:22:04.077059+0000","last_undegraded":"2023-10-09T06:22:04.077059+0000","last_fullsized":"2023-10-09T06:22:04.077059+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093288+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:22:04.093288+0000","last_peered":"2023-10-09T06:22:04.093288+0000","last_clean":"2023-10-09T06:22:04.093288+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:22:04.093288+0000","last_undegraded":"2023-10-09T06:22:04.093288+0000","last_fullsized":"2023-10-09T06:22:04.093288+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.078880+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:22:04.078880+0000","last_peered":"2023-10-09T06:22:04.078880+0000","last_clean":"2023-10-09T06:22:04.078880+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:22:04.078880+0000","last_undegraded":"2023-10-09T06:22:04.078880+0000","last_fullsized":"2023-10-09T06:22:04.078880+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077512+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:22:04.077512+0000","last_peered":"2023-10-09T06:22:04.077512+0000","last_clean":"2023-10-09T06:22:04.077512+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:22:04.077512+0000","last_undegraded":"2023-10-09T06:22:04.077512+0000","last_fullsized":"2023-10-09T06:22:04.077512+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":121,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093204+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:22:04.093204+0000","last_peered":"2023-10-09T06:22:04.093204+0000","last_clean":"2023-10-09T06:22:04.093204+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:22:04.093204+0000","last_undegraded":"2023-10-09T06:22:04.093204+0000","last_fullsized":"2023-10-09T06:22:04.093204+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.096019+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:22:04.096019+0000","last_peered":"2023-10-09T06:22:04.096019+0000","last_clean":"2023-10-09T06:22:04.096019+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:22:04.096019+0000","last_undegraded":"2023-10-09T06:22:04.096019+0000","last_fullsized":"2023-10-09T06:22:04.096019+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093386+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:22:04.093386+0000","last_peered":"2023-10-09T06:22:04.093386+0000","last_clean":"2023-10-09T06:22:04.093386+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:22:04.093386+0000","last_undegraded":"2023-10-09T06:22:04.093386+0000","last_fullsized":"2023-10-09T06:22:04.093386+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":35,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.100424+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:22:04.100424+0000","last_peered":"2023-10-09T06:22:04.100424+0000","last_clean":"2023-10-09T06:22:04.100424+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:22:04.100424+0000","last_undegraded":"2023-10-09T06:22:04.100424+0000","last_fullsized":"2023-10-09T06:22:04.100424+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.090125+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:22:04.090125+0000","last_peered":"2023-10-09T06:22:04.090125+0000","last_clean":"2023-10-09T06:22:04.090125+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:22:04.090125+0000","last_undegraded":"2023-10-09T06:22:04.090125+0000","last_fullsized":"2023-10-09T06:22:04.090125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":147,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.086286+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:22:04.086286+0000","last_peered":"2023-10-09T06:22:04.086286+0000","last_clean":"2023-10-09T06:22:04.086286+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:22:04.086286+0000","last_undegraded":"2023-10-09T06:22:04.086286+0000","last_fullsized":"2023-10-09T06:22:04.086286+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087688+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:22:04.087688+0000","last_peered":"2023-10-09T06:22:04.087688+0000","last_clean":"2023-10-09T06:22:04.087688+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:22:04.087688+0000","last_undegraded":"2023-10-09T06:22:04.087688+0000","last_fullsized":"2023-10-09T06:22:04.087688+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":35,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.086798+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:22:04.086798+0000","last_peered":"2023-10-09T06:22:04.086798+0000","last_clean":"2023-10-09T06:22:04.086798+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:22:04.086798+0000","last_undegraded":"2023-10-09T06:22:04.086798+0000","last_fullsized":"2023-10-09T06:22:04.086798+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":126,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093755+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:22:04.093755+0000","last_peered":"2023-10-09T06:22:04.093755+0000","last_clean":"2023-10-09T06:22:04.093755+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:22:04.093755+0000","last_undegraded":"2023-10-09T06:22:04.093755+0000","last_fullsized":"2023-10-09T06:22:04.093755+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":106,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.134275+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:22:21.134275+0000","last_peered":"2023-10-09T06:22:21.134275+0000","last_clean":"2023-10-09T06:22:21.134275+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:22:21.134275+0000","last_undegraded":"2023-10-09T06:22:21.134275+0000","last_fullsized":"2023-10-09T06:22:21.134275+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.091736+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:22:04.091736+0000","last_peered":"2023-10-09T06:22:04.091736+0000","last_clean":"2023-10-09T06:22:04.091736+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:22:04.091736+0000","last_undegraded":"2023-10-09T06:22:04.091736+0000","last_fullsized":"2023-10-09T06:22:04.091736+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":29,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.089846+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:22:04.089846+0000","last_peered":"2023-10-09T06:22:04.089846+0000","last_clean":"2023-10-09T06:22:04.089846+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:22:04.089846+0000","last_undegraded":"2023-10-09T06:22:04.089846+0000","last_fullsized":"2023-10-09T06:22:04.089846+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":105,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.076988+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:22:04.076988+0000","last_peered":"2023-10-09T06:22:04.076988+0000","last_clean":"2023-10-09T06:22:04.076988+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:22:04.076988+0000","last_undegraded":"2023-10-09T06:22:04.076988+0000","last_fullsized":"2023-10-09T06:22:04.076988+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":111,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077959+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:22:04.077959+0000","last_peered":"2023-10-09T06:22:04.077959+0000","last_clean":"2023-10-09T06:22:04.077959+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:22:04.077959+0000","last_undegraded":"2023-10-09T06:22:04.077959+0000","last_fullsized":"2023-10-09T06:22:04.077959+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":139,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.094432+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:22:04.094432+0000","last_peered":"2023-10-09T06:22:04.094432+0000","last_clean":"2023-10-09T06:22:04.094432+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:22:04.094432+0000","last_undegraded":"2023-10-09T06:22:04.094432+0000","last_fullsized":"2023-10-09T06:22:04.094432+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"137'13118","reported_seq":13164,"reported_epoch":137,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T06:22:25.938522+0000","last_change":"2023-10-09T06:19:01.092933+0000","last_active":"2023-10-09T06:22:25.938522+0000","last_peered":"2023-10-09T06:22:25.938522+0000","last_clean":"2023-10-09T06:18:58.745871+0000","last_became_active":"2023-10-09T06:19:01.079126+0000","last_became_peered":"2023-10-09T06:19:01.079126+0000","last_unstale":"2023-10-09T06:22:25.938522+0000","last_undegraded":"2023-10-09T06:22:25.938522+0000","last_fullsized":"2023-10-09T06:22:25.938522+0000","mapping_epoch":118,"log_start":"121'5400","ondisk_log_start":"121'5400","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:15.303333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4852,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13118,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":-4852}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":33,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087067+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:22:04.087067+0000","last_peered":"2023-10-09T06:22:04.087067+0000","last_clean":"2023-10-09T06:22:04.087067+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:22:04.087067+0000","last_undegraded":"2023-10-09T06:22:04.087067+0000","last_fullsized":"2023-10-09T06:22:04.087067+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087511+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:22:04.087511+0000","last_peered":"2023-10-09T06:22:04.087511+0000","last_clean":"2023-10-09T06:22:04.087511+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:22:04.087511+0000","last_undegraded":"2023-10-09T06:22:04.087511+0000","last_fullsized":"2023-10-09T06:22:04.087511+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"137'12696","reported_seq":12739,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:26.022834+0000","last_change":"2023-10-09T06:17:45.111620+0000","last_active":"2023-10-09T06:22:26.022834+0000","last_peered":"2023-10-09T06:22:26.022834+0000","last_clean":"2023-10-09T06:22:26.022834+0000","last_became_active":"2023-10-09T06:17:45.111092+0000","last_became_peered":"2023-10-09T06:17:45.111092+0000","last_unstale":"2023-10-09T06:22:26.022834+0000","last_undegraded":"2023-10-09T06:22:26.022834+0000","last_fullsized":"2023-10-09T06:22:26.022834+0000","mapping_epoch":105,"log_start":"115'2600","ondisk_log_start":"115'2600","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10096,"log_dups_size":0,"ondisk_log_size":10096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:05.640906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12696,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":33,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.098705+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:22:04.098705+0000","last_peered":"2023-10-09T06:22:04.098705+0000","last_clean":"2023-10-09T06:22:04.098705+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:22:04.098705+0000","last_undegraded":"2023-10-09T06:22:04.098705+0000","last_fullsized":"2023-10-09T06:22:04.098705+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.094062+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:22:04.094062+0000","last_peered":"2023-10-09T06:22:04.094062+0000","last_clean":"2023-10-09T06:22:04.094062+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:22:04.094062+0000","last_undegraded":"2023-10-09T06:22:04.094062+0000","last_fullsized":"2023-10-09T06:22:04.094062+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"137'12910","reported_seq":12953,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:26.008749+0000","last_change":"2023-10-09T06:17:45.115352+0000","last_active":"2023-10-09T06:22:26.008749+0000","last_peered":"2023-10-09T06:22:26.008749+0000","last_clean":"2023-10-09T06:22:26.008749+0000","last_became_active":"2023-10-09T06:17:45.114984+0000","last_became_peered":"2023-10-09T06:17:45.114984+0000","last_unstale":"2023-10-09T06:22:26.008749+0000","last_undegraded":"2023-10-09T06:22:26.008749+0000","last_fullsized":"2023-10-09T06:22:26.008749+0000","mapping_epoch":105,"log_start":"115'2900","ondisk_log_start":"115'2900","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:13.631395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12910,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":33,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.100327+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:22:04.100327+0000","last_peered":"2023-10-09T06:22:04.100327+0000","last_clean":"2023-10-09T06:22:04.100327+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:22:04.100327+0000","last_undegraded":"2023-10-09T06:22:04.100327+0000","last_fullsized":"2023-10-09T06:22:04.100327+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.078492+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:22:04.078492+0000","last_peered":"2023-10-09T06:22:04.078492+0000","last_clean":"2023-10-09T06:22:04.078492+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:22:04.078492+0000","last_undegraded":"2023-10-09T06:22:04.078492+0000","last_fullsized":"2023-10-09T06:22:04.078492+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"137'13242","reported_seq":13285,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:26.003781+0000","last_change":"2023-10-09T06:17:45.120273+0000","last_active":"2023-10-09T06:22:26.003781+0000","last_peered":"2023-10-09T06:22:26.003781+0000","last_clean":"2023-10-09T06:22:26.003781+0000","last_became_active":"2023-10-09T06:17:45.119964+0000","last_became_peered":"2023-10-09T06:17:45.119964+0000","last_unstale":"2023-10-09T06:22:26.003781+0000","last_undegraded":"2023-10-09T06:22:26.003781+0000","last_fullsized":"2023-10-09T06:22:26.003781+0000","mapping_epoch":105,"log_start":"115'3200","ondisk_log_start":"115'3200","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10042,"log_dups_size":0,"ondisk_log_size":10042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:22.213850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13242,"num_write_kb":423744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087957+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:22:04.087957+0000","last_peered":"2023-10-09T06:22:04.087957+0000","last_clean":"2023-10-09T06:22:04.087957+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:22:04.087957+0000","last_undegraded":"2023-10-09T06:22:04.087957+0000","last_fullsized":"2023-10-09T06:22:04.087957+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:13.158197+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:22:13.158197+0000","last_peered":"2023-10-09T06:22:13.158197+0000","last_clean":"2023-10-09T06:22:13.158197+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:22:13.158197+0000","last_undegraded":"2023-10-09T06:22:13.158197+0000","last_fullsized":"2023-10-09T06:22:13.158197+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"137'12988","reported_seq":13048,"reported_epoch":137,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:22:26.014545+0000","last_change":"2023-10-09T06:19:01.121556+0000","last_active":"2023-10-09T06:22:26.014545+0000","last_peered":"2023-10-09T06:22:26.014545+0000","last_clean":"2023-10-09T06:18:58.761034+0000","last_became_active":"2023-10-09T06:19:01.109612+0000","last_became_peered":"2023-10-09T06:19:01.109612+0000","last_unstale":"2023-10-09T06:22:26.014545+0000","last_undegraded":"2023-10-09T06:19:00.825424+0000","last_fullsized":"2023-10-09T06:19:00.781993+0000","mapping_epoch":118,"log_start":"115'2900","ondisk_log_start":"115'2900","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10088,"log_dups_size":0,"ondisk_log_size":10088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:35.553242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4828,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12988,"num_write_kb":415553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"2,5","objects":-4828}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":107,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093390+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:22:04.093390+0000","last_peered":"2023-10-09T06:22:04.093390+0000","last_clean":"2023-10-09T06:22:04.093390+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:22:04.093390+0000","last_undegraded":"2023-10-09T06:22:04.093390+0000","last_fullsized":"2023-10-09T06:22:04.093390+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.086127+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:22:04.086127+0000","last_peered":"2023-10-09T06:22:04.086127+0000","last_clean":"2023-10-09T06:22:04.086127+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:22:04.086127+0000","last_undegraded":"2023-10-09T06:22:04.086127+0000","last_fullsized":"2023-10-09T06:22:04.086127+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"137'12960","reported_seq":13003,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.988339+0000","last_change":"2023-10-09T06:17:45.120554+0000","last_active":"2023-10-09T06:22:25.988339+0000","last_peered":"2023-10-09T06:22:25.988339+0000","last_clean":"2023-10-09T06:22:25.988339+0000","last_became_active":"2023-10-09T06:17:45.120155+0000","last_became_peered":"2023-10-09T06:17:45.120155+0000","last_unstale":"2023-10-09T06:22:25.988339+0000","last_undegraded":"2023-10-09T06:22:25.988339+0000","last_fullsized":"2023-10-09T06:22:25.988339+0000","mapping_epoch":105,"log_start":"121'5200","ondisk_log_start":"121'5200","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:57.941477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12960,"num_write_kb":414720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.078638+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:22:04.078638+0000","last_peered":"2023-10-09T06:22:04.078638+0000","last_clean":"2023-10-09T06:22:04.078638+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:22:04.078638+0000","last_undegraded":"2023-10-09T06:22:04.078638+0000","last_fullsized":"2023-10-09T06:22:04.078638+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087249+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:22:04.087249+0000","last_peered":"2023-10-09T06:22:04.087249+0000","last_clean":"2023-10-09T06:22:04.087249+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:22:04.087249+0000","last_undegraded":"2023-10-09T06:22:04.087249+0000","last_fullsized":"2023-10-09T06:22:04.087249+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"137'13002","reported_seq":13042,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.985883+0000","last_change":"2023-10-09T06:17:45.125233+0000","last_active":"2023-10-09T06:22:25.985883+0000","last_peered":"2023-10-09T06:22:25.985883+0000","last_clean":"2023-10-09T06:22:25.985883+0000","last_became_active":"2023-10-09T06:17:45.124741+0000","last_became_peered":"2023-10-09T06:17:45.124741+0000","last_unstale":"2023-10-09T06:22:25.985883+0000","last_undegraded":"2023-10-09T06:22:25.985883+0000","last_fullsized":"2023-10-09T06:22:25.985883+0000","mapping_epoch":105,"log_start":"115'3000","ondisk_log_start":"115'3000","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13002,"num_write_kb":416064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093871+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:22:04.093871+0000","last_peered":"2023-10-09T06:22:04.093871+0000","last_clean":"2023-10-09T06:22:04.093871+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:22:04.093871+0000","last_undegraded":"2023-10-09T06:22:04.093871+0000","last_fullsized":"2023-10-09T06:22:04.093871+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.091279+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:22:04.091279+0000","last_peered":"2023-10-09T06:22:04.091279+0000","last_clean":"2023-10-09T06:22:04.091279+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:22:04.091279+0000","last_undegraded":"2023-10-09T06:22:04.091279+0000","last_fullsized":"2023-10-09T06:22:04.091279+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"137'12724","reported_seq":12764,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.934478+0000","last_change":"2023-10-09T06:17:45.124190+0000","last_active":"2023-10-09T06:22:25.934478+0000","last_peered":"2023-10-09T06:22:25.934478+0000","last_clean":"2023-10-09T06:22:25.934478+0000","last_became_active":"2023-10-09T06:17:45.123846+0000","last_became_peered":"2023-10-09T06:17:45.123846+0000","last_unstale":"2023-10-09T06:22:25.934478+0000","last_undegraded":"2023-10-09T06:22:25.934478+0000","last_fullsized":"2023-10-09T06:22:25.934478+0000","mapping_epoch":105,"log_start":"121'5500","ondisk_log_start":"121'5500","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":7224,"log_dups_size":0,"ondisk_log_size":7224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12724,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093457+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:22:04.093457+0000","last_peered":"2023-10-09T06:22:04.093457+0000","last_clean":"2023-10-09T06:22:04.093457+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:22:04.093457+0000","last_undegraded":"2023-10-09T06:22:04.093457+0000","last_fullsized":"2023-10-09T06:22:04.093457+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.095150+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:22:04.095150+0000","last_peered":"2023-10-09T06:22:04.095150+0000","last_clean":"2023-10-09T06:22:04.095150+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:22:04.095150+0000","last_undegraded":"2023-10-09T06:22:04.095150+0000","last_fullsized":"2023-10-09T06:22:04.095150+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":948,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:22:04.090668+0000","last_change":"2023-10-09T06:19:00.832644+0000","last_active":"2023-10-09T06:22:04.090668+0000","last_peered":"2023-10-09T06:22:04.090668+0000","last_clean":"2023-10-09T06:18:54.772534+0000","last_became_active":"2023-10-09T06:19:00.832133+0000","last_became_peered":"2023-10-09T06:19:00.832133+0000","last_unstale":"2023-10-09T06:22:04.090668+0000","last_undegraded":"2023-10-09T06:19:00.785661+0000","last_fullsized":"2023-10-09T06:19:00.784414+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:27.419992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0],"avail_no_missing":["1","0","2"],"object_location_counts":[{"shards":"0,1,2","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.091813+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:22:04.091813+0000","last_peered":"2023-10-09T06:22:04.091813+0000","last_clean":"2023-10-09T06:22:04.091813+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:22:04.091813+0000","last_undegraded":"2023-10-09T06:22:04.091813+0000","last_fullsized":"2023-10-09T06:22:04.091813+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.090044+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:22:04.090044+0000","last_peered":"2023-10-09T06:22:04.090044+0000","last_clean":"2023-10-09T06:22:04.090044+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:22:04.090044+0000","last_undegraded":"2023-10-09T06:22:04.090044+0000","last_fullsized":"2023-10-09T06:22:04.090044+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"137'12956","reported_seq":12999,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.998644+0000","last_change":"2023-10-09T06:17:45.113729+0000","last_active":"2023-10-09T06:22:25.998644+0000","last_peered":"2023-10-09T06:22:25.998644+0000","last_clean":"2023-10-09T06:22:25.998644+0000","last_became_active":"2023-10-09T06:17:45.113408+0000","last_became_peered":"2023-10-09T06:17:45.113408+0000","last_unstale":"2023-10-09T06:22:25.998644+0000","last_undegraded":"2023-10-09T06:22:25.998644+0000","last_fullsized":"2023-10-09T06:22:25.998644+0000","mapping_epoch":105,"log_start":"121'4800","ondisk_log_start":"121'4800","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:02.025674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12956,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"137'12902","reported_seq":12945,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.850747+0000","last_change":"2023-10-09T06:17:45.121476+0000","last_active":"2023-10-09T06:22:25.850747+0000","last_peered":"2023-10-09T06:22:25.850747+0000","last_clean":"2023-10-09T06:22:25.850747+0000","last_became_active":"2023-10-09T06:17:45.121043+0000","last_became_peered":"2023-10-09T06:17:45.121043+0000","last_unstale":"2023-10-09T06:22:25.850747+0000","last_undegraded":"2023-10-09T06:22:25.850747+0000","last_fullsized":"2023-10-09T06:22:25.850747+0000","mapping_epoch":105,"log_start":"116'4700","ondisk_log_start":"116'4700","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":8202,"log_dups_size":0,"ondisk_log_size":8202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:39:16.155506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12902,"num_write_kb":412864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":718,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087769+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:22:04.087769+0000","last_peered":"2023-10-09T06:22:04.087769+0000","last_clean":"2023-10-09T06:22:04.087769+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:22:04.087769+0000","last_undegraded":"2023-10-09T06:22:04.087769+0000","last_fullsized":"2023-10-09T06:22:04.087769+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087457+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:22:04.087457+0000","last_peered":"2023-10-09T06:22:04.087457+0000","last_clean":"2023-10-09T06:22:04.087457+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:22:04.087457+0000","last_undegraded":"2023-10-09T06:22:04.087457+0000","last_fullsized":"2023-10-09T06:22:04.087457+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.090868+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:22:04.090868+0000","last_peered":"2023-10-09T06:22:04.090868+0000","last_clean":"2023-10-09T06:22:04.090868+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:22:04.090868+0000","last_undegraded":"2023-10-09T06:22:04.090868+0000","last_fullsized":"2023-10-09T06:22:04.090868+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"137'12648","reported_seq":12707,"reported_epoch":137,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:22:26.012350+0000","last_change":"2023-10-09T06:19:01.101635+0000","last_active":"2023-10-09T06:22:26.012350+0000","last_peered":"2023-10-09T06:22:26.012350+0000","last_clean":"2023-10-09T06:18:58.773534+0000","last_became_active":"2023-10-09T06:19:01.091438+0000","last_became_peered":"2023-10-09T06:19:01.091438+0000","last_unstale":"2023-10-09T06:22:26.012350+0000","last_undegraded":"2023-10-09T06:19:00.824313+0000","last_fullsized":"2023-10-09T06:19:00.781613+0000","mapping_epoch":118,"log_start":"115'2600","ondisk_log_start":"115'2600","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:22.355712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12648,"num_write_kb":404736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,5","objects":-4618}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":496,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.131464+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:22:21.131464+0000","last_peered":"2023-10-09T06:22:21.131464+0000","last_clean":"2023-10-09T06:22:21.131464+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:22:21.131464+0000","last_undegraded":"2023-10-09T06:22:21.131464+0000","last_fullsized":"2023-10-09T06:22:21.131464+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.089690+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:22:04.089690+0000","last_peered":"2023-10-09T06:22:04.089690+0000","last_clean":"2023-10-09T06:22:04.089690+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:22:04.089690+0000","last_undegraded":"2023-10-09T06:22:04.089690+0000","last_fullsized":"2023-10-09T06:22:04.089690+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:21.122733+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:22:21.122733+0000","last_peered":"2023-10-09T06:22:21.122733+0000","last_clean":"2023-10-09T06:22:21.122733+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:22:21.122733+0000","last_undegraded":"2023-10-09T06:22:21.122733+0000","last_fullsized":"2023-10-09T06:22:21.122733+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":485,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.092552+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:22:04.092552+0000","last_peered":"2023-10-09T06:22:04.092552+0000","last_clean":"2023-10-09T06:22:04.092552+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:22:04.092552+0000","last_undegraded":"2023-10-09T06:22:04.092552+0000","last_fullsized":"2023-10-09T06:22:04.092552+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":333,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.086545+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:22:04.086545+0000","last_peered":"2023-10-09T06:22:04.086545+0000","last_clean":"2023-10-09T06:22:04.086545+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:22:04.086545+0000","last_undegraded":"2023-10-09T06:22:04.086545+0000","last_fullsized":"2023-10-09T06:22:04.086545+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.094687+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:22:04.094687+0000","last_peered":"2023-10-09T06:22:04.094687+0000","last_clean":"2023-10-09T06:22:04.094687+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:22:04.094687+0000","last_undegraded":"2023-10-09T06:22:04.094687+0000","last_fullsized":"2023-10-09T06:22:04.094687+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.078527+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:22:04.078527+0000","last_peered":"2023-10-09T06:22:04.078527+0000","last_clean":"2023-10-09T06:22:04.078527+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:22:04.078527+0000","last_undegraded":"2023-10-09T06:22:04.078527+0000","last_fullsized":"2023-10-09T06:22:04.078527+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"137'13148","reported_seq":13194,"reported_epoch":137,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T06:22:25.989083+0000","last_change":"2023-10-09T06:19:01.117941+0000","last_active":"2023-10-09T06:22:25.989083+0000","last_peered":"2023-10-09T06:22:25.989083+0000","last_clean":"2023-10-09T06:18:58.712907+0000","last_became_active":"2023-10-09T06:19:01.104546+0000","last_became_peered":"2023-10-09T06:19:01.104546+0000","last_unstale":"2023-10-09T06:22:25.989083+0000","last_undegraded":"2023-10-09T06:22:25.989083+0000","last_fullsized":"2023-10-09T06:22:25.989083+0000","mapping_epoch":118,"log_start":"121'5000","ondisk_log_start":"121'5000","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:40.858512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4827,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13148,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[0,2,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":-4827}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"137'13078","reported_seq":13118,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.922596+0000","last_change":"2023-10-09T06:17:45.120249+0000","last_active":"2023-10-09T06:22:25.922596+0000","last_peered":"2023-10-09T06:22:25.922596+0000","last_clean":"2023-10-09T06:22:25.922596+0000","last_became_active":"2023-10-09T06:17:45.119883+0000","last_became_peered":"2023-10-09T06:17:45.119883+0000","last_unstale":"2023-10-09T06:22:25.922596+0000","last_undegraded":"2023-10-09T06:22:25.922596+0000","last_fullsized":"2023-10-09T06:22:25.922596+0000","mapping_epoch":105,"log_start":"115'3000","ondisk_log_start":"115'3000","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13078,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":58,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.086361+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:22:04.086361+0000","last_peered":"2023-10-09T06:22:04.086361+0000","last_clean":"2023-10-09T06:22:04.086361+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:22:04.086361+0000","last_undegraded":"2023-10-09T06:22:04.086361+0000","last_fullsized":"2023-10-09T06:22:04.086361+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.076725+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:22:04.076725+0000","last_peered":"2023-10-09T06:22:04.076725+0000","last_clean":"2023-10-09T06:22:04.076725+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:22:04.076725+0000","last_undegraded":"2023-10-09T06:22:04.076725+0000","last_fullsized":"2023-10-09T06:22:04.076725+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077216+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:22:04.077216+0000","last_peered":"2023-10-09T06:22:04.077216+0000","last_clean":"2023-10-09T06:22:04.077216+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:22:04.077216+0000","last_undegraded":"2023-10-09T06:22:04.077216+0000","last_fullsized":"2023-10-09T06:22:04.077216+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":47,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.100233+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:22:04.100233+0000","last_peered":"2023-10-09T06:22:04.100233+0000","last_clean":"2023-10-09T06:22:04.100233+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:22:04.100233+0000","last_undegraded":"2023-10-09T06:22:04.100233+0000","last_fullsized":"2023-10-09T06:22:04.100233+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.090806+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:22:04.090806+0000","last_peered":"2023-10-09T06:22:04.090806+0000","last_clean":"2023-10-09T06:22:04.090806+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:22:04.090806+0000","last_undegraded":"2023-10-09T06:22:04.090806+0000","last_fullsized":"2023-10-09T06:22:04.090806+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.076795+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:22:04.076795+0000","last_peered":"2023-10-09T06:22:04.076795+0000","last_clean":"2023-10-09T06:22:04.076795+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:22:04.076795+0000","last_undegraded":"2023-10-09T06:22:04.076795+0000","last_fullsized":"2023-10-09T06:22:04.076795+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"137'12900","reported_seq":12940,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.993967+0000","last_change":"2023-10-09T06:17:45.120130+0000","last_active":"2023-10-09T06:22:25.993967+0000","last_peered":"2023-10-09T06:22:25.993967+0000","last_clean":"2023-10-09T06:22:25.993967+0000","last_became_active":"2023-10-09T06:17:45.119743+0000","last_became_peered":"2023-10-09T06:17:45.119743+0000","last_unstale":"2023-10-09T06:22:25.993967+0000","last_undegraded":"2023-10-09T06:22:25.993967+0000","last_fullsized":"2023-10-09T06:22:25.993967+0000","mapping_epoch":105,"log_start":"115'3200","ondisk_log_start":"115'3200","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":9700,"log_dups_size":0,"ondisk_log_size":9700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12900,"num_write_kb":412800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":133,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.094897+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:22:04.094897+0000","last_peered":"2023-10-09T06:22:04.094897+0000","last_clean":"2023-10-09T06:22:04.094897+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:22:04.094897+0000","last_undegraded":"2023-10-09T06:22:04.094897+0000","last_fullsized":"2023-10-09T06:22:04.094897+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":441,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:16.704663+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:22:16.704663+0000","last_peered":"2023-10-09T06:22:16.704663+0000","last_clean":"2023-10-09T06:22:16.704663+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:22:16.704663+0000","last_undegraded":"2023-10-09T06:22:16.704663+0000","last_fullsized":"2023-10-09T06:22:16.704663+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":131,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093366+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:22:04.093366+0000","last_peered":"2023-10-09T06:22:04.093366+0000","last_clean":"2023-10-09T06:22:04.093366+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:22:04.093366+0000","last_undegraded":"2023-10-09T06:22:04.093366+0000","last_fullsized":"2023-10-09T06:22:04.093366+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"137'12886","reported_seq":21246,"reported_epoch":137,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:22:34.932002+0000","last_change":"2023-10-09T06:20:05.054322+0000","last_active":"2023-10-09T06:22:34.932002+0000","last_peered":"2023-10-09T06:22:34.932002+0000","last_clean":"2023-10-09T06:18:58.760993+0000","last_became_active":"2023-10-09T06:19:01.088682+0000","last_became_peered":"2023-10-09T06:19:01.088682+0000","last_unstale":"2023-10-09T06:22:34.932002+0000","last_undegraded":"2023-10-09T06:20:05.040625+0000","last_fullsized":"2023-10-09T06:19:00.792897+0000","mapping_epoch":118,"log_start":"122'5700","ondisk_log_start":"122'5700","created":105,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12886,"num_write_kb":412352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4152,"num_bytes_recovered":196018176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"3,4","objects":-635}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":802,"reported_epoch":136,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:22:04.078839+0000","last_change":"2023-10-09T06:19:00.801734+0000","last_active":"2023-10-09T06:22:04.078839+0000","last_peered":"2023-10-09T06:22:04.078839+0000","last_clean":"2023-10-09T06:18:54.789583+0000","last_became_active":"2023-10-09T06:19:00.801207+0000","last_became_peered":"2023-10-09T06:19:00.801207+0000","last_unstale":"2023-10-09T06:22:04.078839+0000","last_undegraded":"2023-10-09T06:19:00.787699+0000","last_fullsized":"2023-10-09T06:19:00.787091+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":92,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077140+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:22:04.077140+0000","last_peered":"2023-10-09T06:22:04.077140+0000","last_clean":"2023-10-09T06:22:04.077140+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:22:04.077140+0000","last_undegraded":"2023-10-09T06:22:04.077140+0000","last_fullsized":"2023-10-09T06:22:04.077140+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093943+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:22:04.093943+0000","last_peered":"2023-10-09T06:22:04.093943+0000","last_clean":"2023-10-09T06:22:04.093943+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:22:04.093943+0000","last_undegraded":"2023-10-09T06:22:04.093943+0000","last_fullsized":"2023-10-09T06:22:04.093943+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"137'13162","reported_seq":13205,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-10-09T06:22:25.914731+0000","last_change":"2023-10-09T06:17:45.119555+0000","last_active":"2023-10-09T06:22:25.914731+0000","last_peered":"2023-10-09T06:22:25.914731+0000","last_clean":"2023-10-09T06:22:25.914731+0000","last_became_active":"2023-10-09T06:17:45.119179+0000","last_became_peered":"2023-10-09T06:17:45.119179+0000","last_unstale":"2023-10-09T06:22:25.914731+0000","last_undegraded":"2023-10-09T06:22:25.914731+0000","last_fullsized":"2023-10-09T06:22:25.914731+0000","mapping_epoch":105,"log_start":"115'4500","ondisk_log_start":"115'4500","created":105,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:44.083388+0000","last_clean_scrub_stamp":"2023-10-09T06:17:44.083388+0000","objects_scrubbed":0,"log_size":8662,"log_dups_size":0,"ondisk_log_size":8662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:15.093934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13162,"num_write_kb":421184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":94,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.076734+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:22:04.076734+0000","last_peered":"2023-10-09T06:22:04.076734+0000","last_clean":"2023-10-09T06:22:04.076734+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:22:04.076734+0000","last_undegraded":"2023-10-09T06:22:04.076734+0000","last_fullsized":"2023-10-09T06:22:04.076734+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.090601+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:22:04.090601+0000","last_peered":"2023-10-09T06:22:04.090601+0000","last_clean":"2023-10-09T06:22:04.090601+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:22:04.090601+0000","last_undegraded":"2023-10-09T06:22:04.090601+0000","last_fullsized":"2023-10-09T06:22:04.090601+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.088102+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:22:04.088102+0000","last_peered":"2023-10-09T06:22:04.088102+0000","last_clean":"2023-10-09T06:22:04.088102+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:22:04.088102+0000","last_undegraded":"2023-10-09T06:22:04.088102+0000","last_fullsized":"2023-10-09T06:22:04.088102+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.087596+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:22:04.087596+0000","last_peered":"2023-10-09T06:22:04.087596+0000","last_clean":"2023-10-09T06:22:04.087596+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:22:04.087596+0000","last_undegraded":"2023-10-09T06:22:04.087596+0000","last_fullsized":"2023-10-09T06:22:04.087596+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":89,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.091901+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:22:04.091901+0000","last_peered":"2023-10-09T06:22:04.091901+0000","last_clean":"2023-10-09T06:22:04.091901+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:22:04.091901+0000","last_undegraded":"2023-10-09T06:22:04.091901+0000","last_fullsized":"2023-10-09T06:22:04.091901+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.089122+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:22:04.089122+0000","last_peered":"2023-10-09T06:22:04.089122+0000","last_clean":"2023-10-09T06:22:04.089122+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:22:04.089122+0000","last_undegraded":"2023-10-09T06:22:04.089122+0000","last_fullsized":"2023-10-09T06:22:04.089122+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":114,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.093328+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:22:04.093328+0000","last_peered":"2023-10-09T06:22:04.093328+0000","last_clean":"2023-10-09T06:22:04.093328+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:22:04.093328+0000","last_undegraded":"2023-10-09T06:22:04.093328+0000","last_fullsized":"2023-10-09T06:22:04.093328+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.094972+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:22:04.094972+0000","last_peered":"2023-10-09T06:22:04.094972+0000","last_clean":"2023-10-09T06:22:04.094972+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:22:04.094972+0000","last_undegraded":"2023-10-09T06:22:04.094972+0000","last_fullsized":"2023-10-09T06:22:04.094972+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":92,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.077437+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:22:04.077437+0000","last_peered":"2023-10-09T06:22:04.077437+0000","last_clean":"2023-10-09T06:22:04.077437+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:22:04.077437+0000","last_undegraded":"2023-10-09T06:22:04.077437+0000","last_fullsized":"2023-10-09T06:22:04.077437+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-10-09T06:22:04.089176+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:22:04.089176+0000","last_peered":"2023-10-09T06:22:04.089176+0000","last_clean":"2023-10-09T06:22:04.089176+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:22:04.089176+0000","last_undegraded":"2023-10-09T06:22:04.089176+0000","last_fullsized":"2023-10-09T06:22:04.089176+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10081,"num_objects_misplaced":9679,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":207320,"num_write_kb":6634177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4152,"num_bytes_recovered":196018176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":932773888,"data_stored":932773888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143120,"ondisk_log_size":143120,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430495,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311416,"kb_used_data":1904,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433736,"statfs":{"total":95995035648,"available":95676145664,"internally_reserved":0,"allocated":1949696,"data_stored":1432328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.4769999999999999,"15min":1.4039999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7969999999999997,"5min":107.36799999999999,"15min":107.36799999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.4359999999999999,"15min":1.391},"min":{"1min":0.371,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.8480000000000008,"5min":105.89,"15min":105.89},"last":0.59999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":2.7799999999999998,"15min":1.514},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":104.825,"15min":104.825},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.3640000000000001,"5min":2.3980000000000001,"15min":1.4179999999999999},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":4.351,"5min":104.637,"15min":104.637},"last":0.61199999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8419999999999996,"5min":4.1180000000000003,"15min":1.992},"min":{"1min":0.46700000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":61.822000000000003,"5min":104.992,"15min":104.992},"last":0.86899999999999999},{"interface":"front","average":{"1min":5.3449999999999998,"5min":4.0529999999999999,"15min":2.012},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":65.064999999999998,"5min":104.691,"15min":104.691},"last":0.57499999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:21:55 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":1.3360000000000001,"15min":1.121},"min":{"1min":0.44,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.411,"5min":1.163,"15min":1.151},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.7519999999999998,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":0.69699999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:21:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8380000000000001,"5min":1.4179999999999999,"15min":1.085},"min":{"1min":0.40500000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9160000000000004,"5min":7.851,"15min":7.851},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.6139999999999999,"5min":1.573,"15min":1.1539999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.6560000000000006,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":0.82299999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:21:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":1.7729999999999999,"15min":1.2629999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":8.3439999999999994,"5min":11.996,"15min":11.996},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.3420000000000001,"5min":1.754,"15min":1.28},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":13.634,"5min":13.634,"15min":13.634},"last":0.38}]},{"osd":6,"last update":"Mon Oct 9 06:21:36 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.3939999999999999,"15min":1.1899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":4.9790000000000001,"5min":9.9510000000000005,"15min":9.9510000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.3500000000000001,"5min":1.516,"15min":1.22},"min":{"1min":0.47799999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":8.6199999999999992,"5min":8.6199999999999992,"15min":8.6199999999999992},"last":0.747}]}]},{"osd":6,"up_from":42,"seq":180388626727,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303208,"kb_used_data":1888,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441944,"statfs":{"total":95995035648,"available":95684550656,"internally_reserved":0,"allocated":1933312,"data_stored":1431571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:33 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.706,"15min":1.1639999999999999},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.1959999999999997,"5min":32.289999999999999,"15min":32.289999999999999},"last":1.099},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.5549999999999999,"15min":1.1359999999999999},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2199999999999998,"5min":31.454999999999998,"15min":31.454999999999998},"last":1.024}]},{"osd":1,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":8.282,"5min":2.605,"15min":1.448},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":137.93899999999999,"5min":137.93899999999999,"15min":137.93899999999999},"last":0.97599999999999998},{"interface":"front","average":{"1min":7.8799999999999999,"5min":2.6429999999999998,"15min":1.476},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":138.02500000000001,"5min":138.02500000000001,"15min":138.02500000000001},"last":0.83499999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:22:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":2.9649999999999999,"15min":1.637},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.952,"5min":95.540000000000006,"15min":95.540000000000006},"last":1.0469999999999999},{"interface":"front","average":{"1min":2.1440000000000001,"5min":3.073,"15min":1.655},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":6.8250000000000002,"5min":96.352000000000004,"15min":96.352000000000004},"last":0.77400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.2030000000000001,"15min":1.1080000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.0650000000000004,"5min":7.3410000000000002,"15min":7.3410000000000002},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.304,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.5220000000000002,"5min":7.5220000000000002,"15min":7.5220000000000002},"last":0.874}]},{"osd":4,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.5509999999999999,"15min":1.135},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.665,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.6679999999999999,"15min":1.212},"min":{"1min":0.35799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.300000000000001,"5min":10.300000000000001,"15min":10.300000000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2480000000000002,"5min":1.802,"15min":1.2729999999999999},"min":{"1min":0.45300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.754,"5min":11.754,"15min":11.754},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.75,"15min":1.284},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.295999999999999,"5min":12.295999999999999,"15min":12.295999999999999},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:21:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.5649999999999999,"15min":1.244},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0389999999999997,"5min":7.0389999999999997,"15min":7.0389999999999997},"last":0.91800000000000004},{"interface":"front","average":{"1min":2.6589999999999998,"5min":1.659,"15min":1.298},"min":{"1min":0.48599999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":9.4139999999999997,"5min":9.4139999999999997,"15min":9.4139999999999997},"last":0.746}]}]},{"osd":1,"up_from":93,"seq":399431958637,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":114096,"kb_used_data":2024,"kb_used_omap":7,"kb_used_meta":112056,"kb_avail":93631056,"statfs":{"total":95995035648,"available":95878201344,"internally_reserved":0,"allocated":2072576,"data_stored":1449035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":114745744},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.427,"15min":0.95099999999999996},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":4.5679999999999996,"5min":31.969000000000001,"15min":31.969000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.5379999999999998,"5min":1.869,"15min":1.079},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.112,"5min":40.667999999999999,"15min":40.667999999999999},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.319,"15min":0.89200000000000002},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.0259999999999998,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.3580000000000001,"15min":1.004},"min":{"1min":0.39600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6959999999999997,"5min":8.3960000000000008,"15min":8.3960000000000008},"last":0.68799999999999994}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.27,"15min":0.97699999999999998},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.9320000000000004,"5min":16.390000000000001,"15min":16.390000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.286,"5min":1.1359999999999999,"15min":0.89200000000000002},"min":{"1min":0.28899999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.1239999999999997,"5min":16.555,"15min":16.555},"last":0.71199999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.3839999999999999,"15min":1.1140000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.0279999999999996,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.774,"5min":1.5660000000000001,"15min":1.1879999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.601,"5min":7.3639999999999999,"15min":7.3639999999999999},"last":1.1419999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.7689999999999999,"15min":1.274},"min":{"1min":0.46999999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.8049999999999997,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.98299999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":1.6819999999999999,"15min":1.258},"min":{"1min":0.47799999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.8860000000000001,"5min":5.6790000000000003,"15min":5.6790000000000003},"last":0.93300000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":1.3799999999999999,"15min":1.1719999999999999},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.6820000000000004,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.966,"5min":1.4730000000000001,"15min":1.2410000000000001},"min":{"1min":0.54700000000000004,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.3609999999999998,"5min":6.3369999999999997,"15min":6.3369999999999997},"last":0.83599999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":1.8700000000000001,"15min":1.375},"min":{"1min":0.57099999999999995,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.806,"5min":7.46,"15min":7.46},"last":0.872},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.5389999999999999,"15min":1.286},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":5.5419999999999998,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":1.204}]}]},{"osd":0,"up_from":88,"seq":377957122162,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787700,"kb_used_data":621484,"kb_used_omap":8,"kb_used_meta":166199,"kb_avail":92957452,"statfs":{"total":95995035648,"available":95188430848,"internally_reserved":0,"allocated":636399616,"data_stored":635771625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":170188032},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.5600000000000001,"15min":0.92600000000000005},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.460000000000001,"5min":12.460000000000001,"15min":12.460000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":2.2370000000000001,"5min":1.4359999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9640000000000004,"5min":10.243,"15min":10.243},"last":1.107}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.532,"15min":0.94999999999999996},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3570000000000002,"5min":13.035,"15min":13.035},"last":0.77600000000000002},{"interface":"front","average":{"1min":2.024,"5min":1.738,"15min":1.0640000000000001},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.006,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.95999999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.1419999999999999,"15min":0.88200000000000001},"min":{"1min":0.248,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5490000000000004,"5min":7.5490000000000004,"15min":7.5490000000000004},"last":1.006},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.056,"15min":0.86499999999999999},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.9610000000000003,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":0.71799999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.706,"15min":1.1879999999999999},"min":{"1min":0.40899999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.3399999999999999,"5min":9.0890000000000004,"15min":9.0890000000000004},"last":1.1379999999999999},{"interface":"front","average":{"1min":2.468,"5min":1.861,"15min":1.2829999999999999},"min":{"1min":0.39600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":10.505000000000001,"5min":10.505000000000001,"15min":10.505000000000001},"last":1.0609999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":1.962,"15min":1.341},"min":{"1min":0.59999999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":7.1760000000000002,"5min":9.548,"15min":9.548},"last":1.194},{"interface":"front","average":{"1min":3.0800000000000001,"5min":1.663,"15min":1.2549999999999999},"min":{"1min":0.40600000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.009,"5min":10.009,"15min":10.009},"last":1.173}]},{"osd":6,"last update":"Mon Oct 9 06:21:42 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.454,"15min":1.1990000000000001},"min":{"1min":0.38800000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.5309999999999997,"5min":10.194000000000001,"15min":10.194000000000001},"last":1.284},{"interface":"front","average":{"1min":1.722,"5min":1.48,"15min":1.165},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.7199999999999998,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":1.262}]},{"osd":7,"last update":"Mon Oct 9 06:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":1.5309999999999999,"15min":1.2749999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":5.6449999999999996,"5min":5.8959999999999999,"15min":5.8959999999999999},"last":1.234},{"interface":"front","average":{"1min":2.363,"5min":1.796,"15min":1.3380000000000001},"min":{"1min":0.45800000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":7.6849999999999996,"5min":7.6849999999999996,"15min":7.6849999999999996},"last":1.3320000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795111,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131496,"kb_used_data":5536,"kb_used_omap":11,"kb_used_meta":125940,"kb_avail":93613656,"statfs":{"total":95995035648,"available":95860383744,"internally_reserved":0,"allocated":5668864,"data_stored":5048906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":128963518},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":2.0150000000000001,"15min":1.135},"min":{"1min":0.224,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.0839999999999996,"5min":35.015999999999998,"15min":35.015999999999998},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.306,"5min":2.3719999999999999,"15min":1.2290000000000001},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":4.492,"5min":36.415999999999997,"15min":36.415999999999997},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.29699999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.6050000000000004,"5min":8.298,"15min":8.298},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.9099999999999999,"15min":1.113},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.0350000000000001,"5min":10.4,"15min":10.4},"last":0.754}]},{"osd":3,"last update":"Mon Oct 9 06:21:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.2110000000000001,"15min":0.89000000000000001},"min":{"1min":0.30099999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.2650000000000006,"5min":8.2650000000000006,"15min":8.2650000000000006},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.2610000000000001,"5min":1.167,"15min":0.85299999999999998},"min":{"1min":0.312,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":18.831,"5min":18.831,"15min":18.831},"last":0.69599999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":2.5899999999999999,"15min":1.502},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":5.7439999999999998,"5min":19.184000000000001,"15min":19.184000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.2090000000000001,"15min":1.363},"min":{"1min":0.41699999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.408000000000001,"5min":29.408000000000001,"15min":29.408000000000001},"last":0.97599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.2320000000000002,"15min":1.417},"min":{"1min":0.39700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":8.0830000000000002,"5min":12.833,"15min":12.833},"last":0.99199999999999999},{"interface":"front","average":{"1min":3.0449999999999999,"5min":1.9770000000000001,"15min":1.2949999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":29.545000000000002,"5min":29.545000000000002,"15min":29.545000000000002},"last":0.93300000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":1.5569999999999999,"15min":1.1910000000000001},"min":{"1min":0.374,"5min":0.372,"15min":0.372},"max":{"1min":29.504000000000001,"5min":29.504000000000001,"15min":29.504000000000001},"last":1.024},{"interface":"front","average":{"1min":1.986,"5min":1.698,"15min":1.234},"min":{"1min":0.46899999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.6760000000000002,"5min":8.1349999999999998,"15min":8.1349999999999998},"last":0.95599999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.827,"15min":1.3380000000000001},"min":{"1min":0.52200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8630000000000004,"5min":8.3309999999999995,"15min":8.3309999999999995},"last":1.069},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.2799999999999998,"15min":1.494},"min":{"1min":0.435,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":4.585,"5min":10.978999999999999,"15min":10.978999999999999},"last":1.0940000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631585,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77208,"kb_used_data":5520,"kb_used_omap":3,"kb_used_meta":71676,"kb_avail":93667944,"statfs":{"total":95995035648,"available":95915974656,"internally_reserved":0,"allocated":5652480,"data_stored":5047116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":73396769},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.996,"15min":1.111},"min":{"1min":0.25,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":9.5739999999999998,"5min":62.939,"15min":62.939},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.5510000000000002,"5min":2.0819999999999999,"15min":1.125},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":7.2789999999999999,"5min":63.869,"15min":63.869},"last":0.68999999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":0.88600000000000001,"15min":0.76800000000000002},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.944,"5min":5.944,"15min":5.944},"last":0.501},{"interface":"front","average":{"1min":1.335,"5min":1.0600000000000001,"15min":0.86299999999999999},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.71999999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":2.2570000000000001,"15min":1.2989999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.891999999999999,"5min":70.594999999999999,"15min":70.594999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.819,"15min":1.095},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.8890000000000002,"5min":62.991999999999997,"15min":62.991999999999997},"last":0.56100000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":1.7529999999999999,"15min":1.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":10.446999999999999,"5min":10.446999999999999,"15min":10.446999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.5190000000000001,"5min":1.5309999999999999,"15min":1.177},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":7.8940000000000001,"5min":7.8940000000000001,"15min":7.8940000000000001},"last":0.70799999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.3300000000000001,"15min":1.1359999999999999},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.9870000000000001,"5min":6.0910000000000002,"15min":6.0910000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.758,"15min":1.304},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":11.454000000000001,"5min":11.754,"15min":11.754},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1160000000000001,"15min":1.119},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.2809999999999999,"15min":1.159},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.0250000000000004,"5min":6.0250000000000004,"15min":6.0250000000000004},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.3799999999999999,"15min":1.1679999999999999},"min":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.4820000000000002,"5min":6.4820000000000002,"15min":6.4820000000000002},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.4630000000000001,"15min":1.2470000000000001},"min":{"1min":0.42099999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":11.439,"5min":11.439,"15min":11.439},"last":0.91000000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019193,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":606384,"kb_used_data":296872,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93138768,"statfs":{"total":95995035648,"available":95374098432,"internally_reserved":0,"allocated":303996928,"data_stored":303487138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.6240000000000001,"15min":1.859},"min":{"1min":0.32500000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.3949999999999996,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.5950000000000002,"5min":3.9300000000000002,"15min":1.994},"min":{"1min":0.39500000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8049999999999997,"5min":84.912999999999997,"15min":84.912999999999997},"last":0.76000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5270000000000001,"5min":2.383,"15min":1.4350000000000001},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":61.726999999999997,"5min":61.726999999999997,"15min":61.726999999999997},"last":0.94999999999999996},{"interface":"front","average":{"1min":5.077,"5min":2.9529999999999998,"15min":1.6399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":61.932000000000002,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.83799999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":3.3559999999999999,"15min":1.726},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.9489999999999998,"5min":61.783999999999999,"15min":61.783999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.9350000000000001,"5min":3.5550000000000002,"15min":1.7789999999999999},"min":{"1min":0.36699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.56,"5min":61.551000000000002,"15min":61.551000000000002},"last":0.69599999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":2.2080000000000002,"15min":1.419},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.217000000000001,"5min":36.154000000000003,"15min":36.154000000000003},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.254,"5min":1.6479999999999999,"15min":1.236},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.4550000000000001,"5min":36.064,"15min":36.064},"last":0.873}]},{"osd":5,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.859,"15min":1.224},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.316},"max":{"1min":12.242000000000001,"5min":12.242000000000001,"15min":12.242000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5720000000000001,"15min":1.5069999999999999},"min":{"1min":0.28499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.234999999999999,"5min":20.113,"15min":20.113},"last":0.51400000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:21:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":2.0299999999999998,"15min":1.3540000000000001},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":18.010000000000002,"5min":18.010000000000002,"15min":18.010000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.716,"15min":1.2350000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.4260000000000002,"5min":13.913,"15min":13.913},"last":0.80200000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:21:34 2023","interfaces":[{"interface":"back","average":{"1min":2.952,"5min":2.0409999999999999,"15min":1.3879999999999999},"min":{"1min":0.46600000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":9.7799999999999994,"5min":15.294,"15min":15.294},"last":0.79100000000000004},{"interface":"front","average":{"1min":2.7690000000000001,"5min":2.0339999999999998,"15min":1.379},"min":{"1min":0.48599999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.176,"5min":10.176,"15min":10.176},"last":0.91200000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822960,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1331200,"data_stored":837023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":3.5710000000000002,"15min":1.8149999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":5.6719999999999997,"5min":69.793999999999997,"15min":69.793999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.774,"5min":2.9969999999999999,"15min":1.5980000000000001},"min":{"1min":0.35599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.0090000000000003,"5min":68.873000000000005,"15min":68.873000000000005},"last":0.54900000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:22:10 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":2.4460000000000002,"15min":1.4399999999999999},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":23.396000000000001,"5min":52.070999999999998,"15min":52.070999999999998},"last":0.79700000000000004},{"interface":"front","average":{"1min":3.5990000000000002,"5min":2.7749999999999999,"15min":1.5269999999999999},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":22.032,"5min":51.570999999999998,"15min":51.570999999999998},"last":0.48999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":4.5620000000000003,"5min":3.1120000000000001,"15min":1.657},"min":{"1min":0.313,"5min":0.307,"15min":0.307},"max":{"1min":45.466000000000001,"5min":47.207000000000001,"15min":47.207000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":4.6390000000000002,"5min":3.1419999999999999,"15min":1.728},"min":{"1min":0.39000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":59.082999999999998,"5min":59.082999999999998,"15min":59.082999999999998},"last":0.70499999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:21:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.746,"15min":1.256},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.0649999999999995,"5min":12.334,"15min":12.334},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.6140000000000001,"5min":1.5800000000000001,"15min":1.218},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.2069999999999999,"5min":11.823,"15min":11.823},"last":0.877}]},{"osd":4,"last update":"Mon Oct 9 06:22:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0899999999999999,"5min":2.6819999999999999,"15min":1.494},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.617000000000001,"5min":17.667999999999999,"15min":17.667999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.4289999999999998,"5min":2.141,"15min":1.3839999999999999},"min":{"1min":0.44700000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.181999999999999,"5min":16.181999999999999,"15min":16.181999999999999},"last":0.746}]},{"osd":6,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":2.0870000000000002,"15min":1.3460000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.7279999999999998,"5min":17.454999999999998,"15min":17.454999999999998},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.75,"5min":1.9890000000000001,"15min":1.3859999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":12.365,"5min":12.365,"15min":12.365},"last":0.82299999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.3540000000000001,"15min":1.5289999999999999},"min":{"1min":0.495,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.3170000000000002,"5min":13.047000000000001,"15min":13.047000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.2130000000000001,"15min":1.4870000000000001},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":10.465999999999999,"5min":11.981,"15min":11.981},"last":0.95099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":630128640,"data_stored":630128640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":302645248,"data_stored":302645248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:22:40.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:22:40.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:40.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:40.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:40.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:40.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:40.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:40.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:40.866 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:22:40.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:40.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:40 smithi125 bash[77299]: audit 2023-10-09T06:22:40.370917+0000 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:40.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:40 smithi125 bash[80002]: audit 2023-10-09T06:22:40.370917+0000 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:40.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:40.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:41.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:41.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:41.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:40 smithi155 bash[59495]: audit 2023-10-09T06:22:40.370917+0000 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:41.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:41.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:41.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:41.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:41.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:41.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:41.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:41.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:41] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T06:22:41.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:41.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:41.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:41.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:41.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:42.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.064 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.772562+0000 mgr.y (mgr.25723) 523 : cluster 0 pgmap v487: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 28 objects/s recovering 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.846341+0000 mon.a (mon.0) 717 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: audit 2023-10-09T06:22:40.856825+0000 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.856904+0000 mon.a (mon.0) 719 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.859064+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.862141+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.897162+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.897236+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.065 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:41 smithi155 bash[59495]: cluster 2023-10-09T06:22:40.897406+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:42.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.772562+0000 mgr.y (mgr.25723) 523 : cluster 0 pgmap v487: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 28 objects/s recovering 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.846341+0000 mon.a (mon.0) 717 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: audit 2023-10-09T06:22:40.856825+0000 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.856904+0000 mon.a (mon.0) 719 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.859064+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.862141+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.897162+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.897236+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[77299]: cluster 2023-10-09T06:22:40.897406+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.772562+0000 mgr.y (mgr.25723) 523 : cluster 0 pgmap v487: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 28 objects/s recovering 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.846341+0000 mon.a (mon.0) 717 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: audit 2023-10-09T06:22:40.856825+0000 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.125:0/2231713341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.856904+0000 mon.a (mon.0) 719 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.859064+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.862141+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.897162+0000 osd.4 (osd.4) 65 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.897236+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:41 smithi125 bash[80002]: cluster 2023-10-09T06:22:40.897406+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e138 with expected crc 2023-10-09T06:22:42.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:42.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:42.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:42.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:42.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:42.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:42.463 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:22:42.464 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:22:42.464 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:22:42.464 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:22:42.464 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:22:42.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:42.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:42.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:42.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:42.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:42.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:42.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:42.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:42.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:43.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:43.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:43.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:43.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:43.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:43.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:43.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:22:43.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:43.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:22:43.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:43.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:43.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:43.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:43.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:43.803 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:43.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:43.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:43.918 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:43 smithi125 bash[77299]: cluster 2023-10-09T06:22:42.773408+0000 mgr.y (mgr.25723) 524 : cluster 0 pgmap v489: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:43.918 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:43 smithi125 bash[77299]: cluster 2023-10-09T06:22:43.666561+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 9987/84 objects degraded (11889.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:43.918 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:43 smithi125 bash[80002]: cluster 2023-10-09T06:22:42.773408+0000 mgr.y (mgr.25723) 524 : cluster 0 pgmap v489: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:43.919 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:43 smithi125 bash[80002]: cluster 2023-10-09T06:22:43.666561+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 9987/84 objects degraded (11889.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:43.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:43.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:43.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:44.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:44.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:43 smithi155 bash[59495]: cluster 2023-10-09T06:22:42.773408+0000 mgr.y (mgr.25723) 524 : cluster 0 pgmap v489: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:44.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:43 smithi155 bash[59495]: cluster 2023-10-09T06:22:43.666561+0000 mon.a (mon.0) 720 : cluster 3 Health check update: Degraded data redundancy: 9987/84 objects degraded (11889.286%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:22:44.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:44.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:44.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:44.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:44.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:44.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:44.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:44.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:44.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:44.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:44.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:44.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:22:44.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:44.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:44.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:44.880 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_0' removed 2023-10-09T06:22:44.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:45.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:45.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:45.136 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:44 smithi155 bash[59495]: audit 2023-10-09T06:22:44.543046+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:44 smithi125 bash[80002]: audit 2023-10-09T06:22:44.543046+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:45.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:44 smithi125 bash[77299]: audit 2023-10-09T06:22:44.543046+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:45.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:45.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:45.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:45.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:45.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:45.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:45.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:45.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:45.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:45.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:45.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:45.863 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:22:45.864 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:22:45.864 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T06:22:45.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T06:22:45.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:45.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.774349+0000 mgr.y (mgr.25723) 525 : cluster 0 pgmap v490: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:45.945 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: audit 2023-10-09T06:22:44.864765+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:45.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.864829+0000 mon.a (mon.0) 723 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:22:45.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.866369+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:45.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.898934+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:45.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.899009+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:45.946 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:45 smithi155 bash[59495]: cluster 2023-10-09T06:22:44.899247+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:45.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:45.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:46.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:46.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:46.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.774349+0000 mgr.y (mgr.25723) 525 : cluster 0 pgmap v490: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:46.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: audit 2023-10-09T06:22:44.864765+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:46.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.864829+0000 mon.a (mon.0) 723 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:22:46.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.866369+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.898934+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.899009+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[80002]: cluster 2023-10-09T06:22:44.899247+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.774349+0000 mgr.y (mgr.25723) 525 : cluster 0 pgmap v490: 89 pgs: 2 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9987/84 objects degraded (11889.286%); 9679/84 objects misplaced (11522.619%); 0 B/s, 23 objects/s recovering 2023-10-09T06:22:46.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: audit 2023-10-09T06:22:44.864765+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.125:0/1150355775' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:46.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.864829+0000 mon.a (mon.0) 723 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:22:46.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.866369+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.898934+0000 osd.4 (osd.4) 66 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.899009+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.144 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:45 smithi125 bash[77299]: cluster 2023-10-09T06:22:44.899247+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e139 with expected crc 2023-10-09T06:22:46.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:46.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:46.348 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:46.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:46.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:46.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:46.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:46.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:46.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:45.872270+0000 mon.a (mon.0) 724 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:45.878007+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:45.880277+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:45.881208+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.141019+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:46.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.141087+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:46.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:46.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:47.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:47.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:45.872270+0000 mon.a (mon.0) 724 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:22:47.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:45.878007+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:45.880277+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:45.881208+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.141019+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.076 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:46 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.141087+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:47.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:45.872270+0000 mon.a (mon.0) 724 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:22:47.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:45.878007+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:45.880277+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:45.881208+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.141019+0000 osd.4 (osd.4) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:46 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.141087+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e140 with expected crc 2023-10-09T06:22:47.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:47.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:47.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:47.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:47.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:47.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:47.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:47.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:47.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:47.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:47.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:47.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:47.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:47.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.775178+0000 mgr.y (mgr.25723) 526 : cluster 0 pgmap v493: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.876401+0000 mon.a (mon.0) 725 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.877732+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.881311+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.882633+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:46.884968+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:47.056142+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.001 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:47 smithi155 bash[59495]: cluster 2023-10-09T06:22:47.056233+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:48.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:48.133 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.775178+0000 mgr.y (mgr.25723) 526 : cluster 0 pgmap v493: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.876401+0000 mon.a (mon.0) 725 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.877732+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.881311+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.882633+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:46.884968+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:47.056142+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[77299]: cluster 2023-10-09T06:22:47.056233+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.134 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.775178+0000 mgr.y (mgr.25723) 526 : cluster 0 pgmap v493: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.876401+0000 mon.a (mon.0) 725 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.877732+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.881311+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.882633+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:46.884968+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:47.056142+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:47 smithi125 bash[80002]: cluster 2023-10-09T06:22:47.056233+0000 osd.4 (osd.4) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-09T06:22:48.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:48.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:48.235 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:48.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:48.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:48.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:48.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:48.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:48.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:48.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:48.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:48.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:48.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:48.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:48.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:48 smithi125 bash[80002]: audit 2023-10-09T06:22:48.571197+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:48.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:48 smithi125 bash[80002]: cluster 2023-10-09T06:22:48.667779+0000 mon.a (mon.0) 727 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:22:48.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:48.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:48.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:49.034 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:49.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:49.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:48 smithi125 bash[77299]: audit 2023-10-09T06:22:48.571197+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:49.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:48 smithi125 bash[77299]: cluster 2023-10-09T06:22:48.667779+0000 mon.a (mon.0) 727 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:22:49.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:48 smithi155 bash[59495]: audit 2023-10-09T06:22:48.571197+0000 mon.a (mon.0) 726 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:49.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:48 smithi155 bash[59495]: cluster 2023-10-09T06:22:48.667779+0000 mon.a (mon.0) 727 : cluster 3 Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:22:49.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:49.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:49.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:49.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:49.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:49.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:49.398 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:49.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:49.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:49.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:49.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:49.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:49.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:49.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:49.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:49.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:49.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:49.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:49.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:50.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:50.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:50.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:49 smithi155 bash[59495]: cluster 2023-10-09T06:22:48.775780+0000 mgr.y (mgr.25723) 527 : cluster 0 pgmap v495: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:50.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:50.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:50.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:49 smithi125 bash[77299]: cluster 2023-10-09T06:22:48.775780+0000 mgr.y (mgr.25723) 527 : cluster 0 pgmap v495: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:50.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:49 smithi125 bash[80002]: cluster 2023-10-09T06:22:48.775780+0000 mgr.y (mgr.25723) 527 : cluster 0 pgmap v495: 73 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10/84 objects degraded (11.905%) 2023-10-09T06:22:50.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:50.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:50.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:50.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:50.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:50.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:50.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:50.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:50.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:50.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:50.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:50.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:50.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:50.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:51.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:51.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:50 smithi155 bash[59495]: cluster 2023-10-09T06:22:50.885209+0000 mon.a (mon.0) 728 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:22:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:51.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:51.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:50 smithi125 bash[77299]: cluster 2023-10-09T06:22:50.885209+0000 mon.a (mon.0) 728 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:22:51.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:50 smithi125 bash[80002]: cluster 2023-10-09T06:22:50.885209+0000 mon.a (mon.0) 728 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:22:51.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:51.249 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:51.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:51.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:51.427 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:51.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:51.505 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:22:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:22:51] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-10-09T06:22:51.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:51.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:51.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:51.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:51.803 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:51.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:51.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:51.836 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:22:51.838 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:22:51.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.914 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:19398 5.2 5:4bc34fa9:::benchmark_data_smithi125_106927_object19397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:16.748664+0000", 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 275.153790351, 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.081957260000000004, 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 19398 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.915 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748664+0000", 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748664+0000", 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748662+0000", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8603999999999997e-05 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748711+0000", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1715999999999999e-05 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748722+0000", 2023-10-09T06:22:51.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1949999999999997e-06 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748730+0000", 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3684000000000003e-05 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.748783+0000", 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00029033599999999998 2023-10-09T06:22:51.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.749074+0000", 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015128299999999999 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.749225+0000", 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001510171 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.750735+0000", 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001136001 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.751871+0000", 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.078538584999999994 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.830410+0000", 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013793400000000001 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.830548+0000", 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3219000000000005e-05 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:16.830621+0000", 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.081957260000000004 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:31592 5.2 5:4601397b:::benchmark_data_smithi125_106927_object31591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e114)", 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:23.260268+0000", 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 268.642185509, 2023-10-09T06:22:51.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.076272171, 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 31592 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260268+0000", 2023-10-09T06:22:51.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260268+0000", 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260267+0000", 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4042000000000005e-05 2023-10-09T06:22:51.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260361+0000", 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7608000000000001e-05 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260378+0000", 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6930000000000007e-06 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260386+0000", 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6364000000000004e-05 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260452+0000", 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015640900000000001 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260609+0000", 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034077499999999998 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.260949+0000", 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027764069999999998 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.263726+0000", 2023-10-09T06:22:51.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037282599999999998 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.264099+0000", 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.072123111000000004 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.336222+0000", 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017696000000000001 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.336399+0000", 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014185099999999999 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:23.336541+0000", 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.076272171 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.929 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:85005 5.2 5:43f56a72:::benchmark_data_smithi125_106927_object85004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:11.600414+0000", 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 220.30204026199999, 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093137191999999994, 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.930 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 85005 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600414+0000", 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600414+0000", 2023-10-09T06:22:51.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600412+0000", 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6493000000000001e-05 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600488+0000", 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2977e-05 2023-10-09T06:22:51.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600501+0000", 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6449999999999999e-06 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600508+0000", 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026394199999999997 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600772+0000", 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015883600000000001 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.600930+0000", 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021193000000000001 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.601142+0000", 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066976919999999999 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.607840+0000", 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01415144 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.621992+0000", 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071449902999999995 2023-10-09T06:22:51.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693441+0000", 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6147000000000002e-05 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693508+0000", 2023-10-09T06:22:51.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3297999999999998e-05 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693551+0000", 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.093137191999999994 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.937 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:85008 5.2 5:4ef48e81:::benchmark_data_smithi125_106927_object85007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:11.602264+0000", 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 220.30018981800001, 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091660228999999996, 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 85008 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.938 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602264+0000", 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602264+0000", 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602262+0000", 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1856e-05 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602304+0000", 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8936999999999998e-05 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602323+0000", 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6770000000000001e-06 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.602332+0000", 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022170670000000001 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.604549+0000", 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017303899999999999 2023-10-09T06:22:51.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.604722+0000", 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021623899999999999 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.604938+0000", 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032676670000000001 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.608206+0000", 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016938482000000001 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.625144+0000", 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.068559561000000005 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693704+0000", 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121069 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693825+0000", 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9578000000000001e-05 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:11.693924+0000", 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091660228999999996 2023-10-09T06:22:51.944 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:86972 5.2 5:4da22672:::benchmark_data_smithi125_106927_object86971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:14.485034+0000", 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 217.417419572, 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21526514299999999, 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.945 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 86972 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485034+0000", 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485034+0000", 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485033+0000", 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113489 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485146+0000", 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1699e-05 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485168+0000", 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5320000000000003e-06 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.485174+0000", 2023-10-09T06:22:51.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053826840000000004 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.490557+0000", 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00040561900000000002 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.490963+0000", 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00053335899999999996 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.491496+0000", 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0035083620000000001 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.495004+0000", 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083750070000000003 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.503379+0000", 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.196825529 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700205+0000", 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3899e-05 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700259+0000", 2023-10-09T06:22:51.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0624999999999998e-05 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700300+0000", 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21526514299999999 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:86977 5.2 5:423bd8b3:::benchmark_data_smithi125_106927_object86976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:22:51.952 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:14.497520+0000", 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 217.40493378100001, 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20289372, 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 86977 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497520+0000", 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497520+0000", 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497518+0000", 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5058999999999997e-05 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497563+0000", 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6714999999999998e-05 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497579+0000", 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7300000000000005e-06 2023-10-09T06:22:51.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497587+0000", 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2824000000000001e-05 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497640+0000", 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000163394 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.497803+0000", 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019623000000000001 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.498000+0000", 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083038109999999995 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.506303+0000", 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019921700000000001 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.508296+0000", 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19203880400000001 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700334+0000", 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7564000000000002e-05 2023-10-09T06:22:51.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700372+0000", 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1983000000000001e-05 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700414+0000", 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20289372 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:86994 5.2 5:40199408:::benchmark_data_smithi125_106927_object86993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:14.532363+0000", 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 217.370090648, 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16817274199999999, 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 86994 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532363+0000", 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532363+0000", 2023-10-09T06:22:51.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532361+0000", 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2614000000000001e-05 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532413+0000", 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4117999999999998e-05 2023-10-09T06:22:51.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532437+0000", 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.747e-06 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532447+0000", 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0777000000000003e-05 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532528+0000", 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023697900000000001 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.532765+0000", 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030721699999999998 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.533072+0000", 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043055280000000003 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.537378+0000", 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000190566 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.537568+0000", 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16288166800000001 2023-10-09T06:22:51.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700450+0000", 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4574999999999998e-05 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700504+0000", 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1708000000000002e-05 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.700536+0000", 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16817274199999999 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:116631 5.2 5:4b63a802:::benchmark_data_smithi125_106927_object12969:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:57.496323+0000", 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 174.406130741, 2023-10-09T06:22:51.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095833506999999998, 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 116631 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.968 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496323+0000", 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496323+0000", 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496321+0000", 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2431e-05 2023-10-09T06:22:51.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496333+0000", 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8431e-05 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496351+0000", 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9469999999999998e-06 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496361+0000", 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8133000000000006e-05 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496429+0000", 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7663000000000003e-05 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496527+0000", 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118799 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.496646+0000", 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034584080000000001 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.500104+0000", 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7273000000000001e-05 2023-10-09T06:22:51.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.500132+0000", 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.091882121999999997 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.592014+0000", 2023-10-09T06:22:51.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010071 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.592114+0000", 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2268999999999998e-05 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:57.592157+0000", 2023-10-09T06:22:51.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.095833506999999998 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:117058 5.2 5:4dc39e70:::benchmark_data_smithi125_106927_object13396:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:58.197761+0000", 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 173.70469339100001, 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065843347999999996, 2023-10-09T06:22:51.975 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117058 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197761+0000", 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197761+0000", 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999905 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197751+0000", 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0513000000000002e-05 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197771+0000", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8023e-05 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197789+0000", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0368e-05 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197800+0000", 2023-10-09T06:22:51.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6088999999999998e-05 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.197856+0000", 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015453400000000001 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.198010+0000", 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000179619 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.979 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.198190+0000", 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0079005380000000004 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.206091+0000", 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048601399999999998 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.210951+0000", 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052435636000000001 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.263386+0000", 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116235 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.263503+0000", 2023-10-09T06:22:51.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000101338 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.263604+0000", 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.065843347999999996 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:121057 5.2 5:43ef41c2:::benchmark_data_smithi125_106927_object17395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:51.984 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:04.283449+0000", 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 167.61900513800001, 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066839157999999996, 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 121057 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283449+0000", 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283449+0000", 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283447+0000", 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.764e-06 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283457+0000", 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5062000000000001e-05 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283472+0000", 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9369999999999998e-06 2023-10-09T06:22:51.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283479+0000", 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7194000000000002e-05 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283526+0000", 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014778 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283674+0000", 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015069899999999999 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.283824+0000", 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064183779999999998 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.290243+0000", 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001516512 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.291759+0000", 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.058266983000000001 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.350026+0000", 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000149087 2023-10-09T06:22:51.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.350175+0000", 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112754 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:04.350288+0000", 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066839157999999996 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.991 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:122657 5.2 5:4e203f22:::benchmark_data_smithi125_106927_object18995:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:06.778498+0000", 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 165.123955448, 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075001349999999994, 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:51.992 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122657 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778498+0000", 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778498+0000", 2023-10-09T06:22:51.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778496+0000", 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4295e-05 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778510+0000", 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8260000000000001e-05 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778528+0000", 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8929999999999994e-06 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778537+0000", 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014937199999999999 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:51.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778687+0000", 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019996200000000001 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.778887+0000", 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116322 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.779003+0000", 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043611320000000002 2023-10-09T06:22:51.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.783364+0000", 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016889300000000001 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.783533+0000", 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069865393999999997 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.853398+0000", 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8421000000000002e-05 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.853457+0000", 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3137999999999999e-05 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:51.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:06.853500+0000", 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075001349999999994 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:122981 5.2 5:4b3c65dd:::benchmark_data_smithi125_106927_object19319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:07.291292+0000", 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 164.6111621, 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088290075999999995, 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:51.999 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122981 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291292+0000", 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291292+0000", 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291290+0000", 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4269999999999997e-06 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291299+0000", 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4749999999999999e-05 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291314+0000", 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0249999999999997e-06 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291321+0000", 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3018e-05 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291374+0000", 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111421 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291486+0000", 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119667 2023-10-09T06:22:52.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.291605+0000", 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012585697 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.304191+0000", 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00049081500000000002 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.004 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.304682+0000", 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.074769295 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379451+0000", 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6808000000000001e-05 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379528+0000", 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3958000000000001e-05 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379582+0000", 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.088290075999999995 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.006 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:122984 5.2 5:4843c170:::benchmark_data_smithi125_106927_object19322:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:07.294803+0000", 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 164.60765065300001, 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.085015640000000003, 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 122984 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294803+0000", 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294803+0000", 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294802+0000", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.14e-06 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294810+0000", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6025e-05 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294826+0000", 2023-10-09T06:22:52.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9110000000000004e-06 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.294833+0000", 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049289929999999996 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.299762+0000", 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014359400000000001 2023-10-09T06:22:52.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.299905+0000", 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103355 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.300009+0000", 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004658616 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.304667+0000", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0122000000000001e-05 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.304687+0000", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075025372000000007 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379713+0000", 2023-10-09T06:22:52.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7987e-05 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379761+0000", 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8294999999999997e-05 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:07.379819+0000", 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.085015640000000003 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.013 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:147177 5.2 5:47b523b1:::benchmark_data_smithi125_106927_object43515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:46.062113+0000", 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 125.840340537, 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066198570999999998, 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.014 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.015 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.015 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 147177 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062113+0000", 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062113+0000", 2023-10-09T06:22:52.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062111+0000", 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0017000000000001e-05 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062121+0000", 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0372000000000001e-05 2023-10-09T06:22:52.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062142+0000", 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6009999999999997e-06 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062149+0000", 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014883400000000001 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062298+0000", 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014019 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062438+0000", 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015045299999999999 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.062589+0000", 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0086970469999999994 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.071286+0000", 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001744046 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.073030+0000", 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.055027006000000003 2023-10-09T06:22:52.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.128057+0000", 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014847699999999999 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.128205+0000", 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106691 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:46.128312+0000", 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066198570999999998 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:151332 5.2 5:4b1f472f:::benchmark_data_smithi125_106927_object47670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:52.878482+0000", 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 119.023972224, 2023-10-09T06:22:52.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075729228999999995, 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 151332 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878482+0000", 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878482+0000", 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878480+0000", 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0617e-05 2023-10-09T06:22:52.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878490+0000", 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8553e-05 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878509+0000", 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7130000000000005e-06 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878516+0000", 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4028000000000004e-05 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878580+0000", 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014523199999999999 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878726+0000", 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000149462 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.878875+0000", 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051594040000000002 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.884035+0000", 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8619999999999997e-06 2023-10-09T06:22:52.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.884041+0000", 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069911607000000001 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.953953+0000", 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015135999999999999 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.954104+0000", 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106578 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:52.954211+0000", 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.075729228999999995 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:198410 5.2 5:48e5d81e:::benchmark_data_smithi125_106927_object94748:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:10.896002+0000", 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 41.006452113999998, 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11146840600000001, 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 198410 2023-10-09T06:22:52.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.896002+0000", 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.896002+0000", 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.895999+0000", 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1321999999999999e-05 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.896010+0000", 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9576999999999999e-05 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.032 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.896030+0000", 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0429999999999996e-06 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.896039+0000", 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033922660000000001 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.899431+0000", 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031374199999999999 2023-10-09T06:22:52.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.899745+0000", 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00043742899999999998 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.900183+0000", 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005481784 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.905664+0000", 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042354200000000002 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:10.909900+0000", 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.097428871 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.007329+0000", 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6798999999999993e-05 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.007415+0000", 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4829e-05 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.007470+0000", 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11146840600000001 2023-10-09T06:22:52.036 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:198685 5.2 5:4f7f18ba:::benchmark_data_smithi125_106927_object95023:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:11.405071+0000", 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 40.497382641999998, 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.104551122, 2023-10-09T06:22:52.037 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 198685 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405071+0000", 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405071+0000", 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405069+0000", 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2616000000000001e-05 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405082+0000", 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0721e-05 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405102+0000", 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3540000000000006e-06 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405112+0000", 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5794000000000001e-05 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405167+0000", 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013635399999999999 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405304+0000", 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014683200000000001 2023-10-09T06:22:52.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.405451+0000", 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053264269999999999 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.410777+0000", 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014755899999999999 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.410925+0000", 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098588446999999996 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.509513+0000", 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1679000000000004e-05 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.509575+0000", 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7698999999999999e-05 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.509622+0000", 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.104551122 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.044 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:198983 5.2 5:4a988c16:::benchmark_data_smithi125_106927_object95321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:11.959325+0000", 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 39.943129233999997, 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.073888983000000005, 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 198983 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959325+0000", 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959325+0000", 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959323+0000", 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5189999999999998e-06 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959332+0000", 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5665e-05 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959348+0000", 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3939999999999998e-06 2023-10-09T06:22:52.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959355+0000", 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3708000000000002e-05 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959399+0000", 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3423000000000003e-05 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959492+0000", 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4036000000000006e-05 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.959586+0000", 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041867550000000003 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.963773+0000", 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005339992 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.969113+0000", 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.063982591000000005 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033096+0000", 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9635000000000007e-05 2023-10-09T06:22:52.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033165+0000", 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8409999999999999e-05 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033214+0000", 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.073888983000000005 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.051 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:198985 5.2 5:47cdc180:::benchmark_data_smithi125_106927_object95323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:11.964243+0000", 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 39.938211353, 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069102854000000005, 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.052 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 198985 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964243+0000", 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964243+0000", 2023-10-09T06:22:52.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964241+0000", 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9130000000000003e-06 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964249+0000", 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2578999999999999e-05 2023-10-09T06:22:52.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964261+0000", 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.229e-06 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964268+0000", 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8656999999999998e-05 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964316+0000", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012874300000000001 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964445+0000", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014592000000000001 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.964591+0000", 2023-10-09T06:22:52.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048405499999999999 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.969431+0000", 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8583e-05 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:11.969470+0000", 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.063785843999999994 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033256+0000", 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8406999999999999e-05 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033304+0000", 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1151e-05 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.033345+0000", 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.069102854000000005 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:199375 5.2 5:4a709da4:::benchmark_data_smithi125_106927_object95713:head [delete] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:12.673149+0000", 2023-10-09T06:22:52.059 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 39.229304972999998, 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082059213000000006, 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 199375 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673149+0000", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673149+0000", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673147+0000", 2023-10-09T06:22:52.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1034e-05 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673158+0000", 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9596999999999999e-05 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673177+0000", 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0974e-05 2023-10-09T06:22:52.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673188+0000", 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0436999999999998e-05 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673249+0000", 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000143778 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673392+0000", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014066999999999999 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.673533+0000", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052562149999999998 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.678789+0000", 2023-10-09T06:22:52.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003140608 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.681930+0000", 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.073176049000000007 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.755106+0000", 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9073000000000003e-05 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.755155+0000", 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3193999999999999e-05 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:12.755208+0000", 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082059213000000006 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.066 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:22:52.067 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:22:52.067 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:22:52.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:52.129 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:51 smithi125 bash[80002]: cluster 2023-10-09T06:22:50.776618+0000 mgr.y (mgr.25723) 528 : cluster 0 pgmap v496: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:22:52.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:51 smithi125 bash[77299]: cluster 2023-10-09T06:22:50.776618+0000 mgr.y (mgr.25723) 528 : cluster 0 pgmap v496: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:22:52.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:52.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:52.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:52.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:51 smithi155 bash[59495]: cluster 2023-10-09T06:22:50.776618+0000 mgr.y (mgr.25723) 528 : cluster 0 pgmap v496: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:22:52.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:52.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:52.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:52.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:52.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:52.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:52.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:52.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:22:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:52.868 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:52.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:52.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:52.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:53.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:53.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:53.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:53.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:52 smithi155 bash[59495]: audit 2023-10-09T06:22:51.803291+0000 mgr.y (mgr.25723) 529 : audit 0 from='client.44882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:52 smithi155 bash[59495]: audit 2023-10-09T06:22:51.944433+0000 mon.a (mon.0) 729 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:52 smithi155 bash[59495]: audit 2023-10-09T06:22:52.317501+0000 mon.a (mon.0) 730 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:52 smithi155 bash[59495]: audit 2023-10-09T06:22:52.323607+0000 mon.a (mon.0) 731 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.156 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[77299]: audit 2023-10-09T06:22:51.803291+0000 mgr.y (mgr.25723) 529 : audit 0 from='client.44882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[77299]: audit 2023-10-09T06:22:51.944433+0000 mon.a (mon.0) 729 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[77299]: audit 2023-10-09T06:22:52.317501+0000 mon.a (mon.0) 730 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[77299]: audit 2023-10-09T06:22:52.323607+0000 mon.a (mon.0) 731 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[80002]: audit 2023-10-09T06:22:51.803291+0000 mgr.y (mgr.25723) 529 : audit 0 from='client.44882 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[80002]: audit 2023-10-09T06:22:51.944433+0000 mon.a (mon.0) 729 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[80002]: audit 2023-10-09T06:22:52.317501+0000 mon.a (mon.0) 730 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:52 smithi125 bash[80002]: audit 2023-10-09T06:22:52.323607+0000 mon.a (mon.0) 731 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:53.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:53.287 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:53.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:53.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:53.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:53.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:53.555 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:53.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:53.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:53.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:53.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:53.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:53.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:53.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:54.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:54.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:54.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:54.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:53 smithi155 bash[59495]: cluster 2023-10-09T06:22:52.777448+0000 mgr.y (mgr.25723) 530 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:54.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:53 smithi125 bash[77299]: cluster 2023-10-09T06:22:52.777448+0000 mgr.y (mgr.25723) 530 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:54.202 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:53 smithi125 bash[80002]: cluster 2023-10-09T06:22:52.777448+0000 mgr.y (mgr.25723) 530 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:54.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:54.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:54.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:54.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:54.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:54.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:54.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:54.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:54.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:54.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:54.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:54.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:55.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:55.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:55.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:55.407 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:55.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:55.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:55.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:55.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:55.676 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:55.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:55.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:55.885 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":496,"stamp":"2023-10-09T06:22:50.776024+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":1190420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":233,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2618672,"kb_used_data":915312,"kb_used_omap":30,"kb_used_meta":1703265,"kb_avail":747342544,"statfs":{"total":767960285184,"available":765278765056,"internally_reserved":0,"allocated":937279488,"data_stored":932695714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1744143471},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":41,"apply_latency_ms":41,"commit_latency_ns":41000000,"apply_latency_ns":41000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-9987,"num_objects_misplaced":-9679,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-207320,"num_write_kb":-6634177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4246,"num_bytes_recovered":-196018176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002355"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":122,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147303+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:22:46.147303+0000","last_peered":"2023-10-09T06:22:46.147303+0000","last_clean":"2023-10-09T06:22:46.147303+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:22:46.147303+0000","last_undegraded":"2023-10-09T06:22:46.147303+0000","last_fullsized":"2023-10-09T06:22:46.147303+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":110,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856873+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:22:40.856873+0000","last_peered":"2023-10-09T06:22:40.856873+0000","last_clean":"2023-10-09T06:22:40.856873+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:22:40.856873+0000","last_undegraded":"2023-10-09T06:22:40.856873+0000","last_fullsized":"2023-10-09T06:22:40.856873+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.904104+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:22:44.904104+0000","last_peered":"2023-10-09T06:22:44.904104+0000","last_clean":"2023-10-09T06:22:44.904104+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:22:44.904104+0000","last_undegraded":"2023-10-09T06:22:44.904104+0000","last_fullsized":"2023-10-09T06:22:44.904104+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":119,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877381+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:22:45.877381+0000","last_peered":"2023-10-09T06:22:45.877381+0000","last_clean":"2023-10-09T06:22:45.877381+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:22:45.877381+0000","last_undegraded":"2023-10-09T06:22:45.877381+0000","last_fullsized":"2023-10-09T06:22:45.877381+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855764+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:22:40.855764+0000","last_peered":"2023-10-09T06:22:40.855764+0000","last_clean":"2023-10-09T06:22:40.855764+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:22:40.855764+0000","last_undegraded":"2023-10-09T06:22:40.855764+0000","last_fullsized":"2023-10-09T06:22:40.855764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.886698+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:22:45.886698+0000","last_peered":"2023-10-09T06:22:45.886698+0000","last_clean":"2023-10-09T06:22:45.886698+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:22:45.886698+0000","last_undegraded":"2023-10-09T06:22:45.886698+0000","last_fullsized":"2023-10-09T06:22:45.886698+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877809+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:22:45.877809+0000","last_peered":"2023-10-09T06:22:45.877809+0000","last_clean":"2023-10-09T06:22:45.877809+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:22:45.877809+0000","last_undegraded":"2023-10-09T06:22:45.877809+0000","last_fullsized":"2023-10-09T06:22:45.877809+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":123,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855641+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:22:40.855641+0000","last_peered":"2023-10-09T06:22:40.855641+0000","last_clean":"2023-10-09T06:22:40.855641+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:22:40.855641+0000","last_undegraded":"2023-10-09T06:22:40.855641+0000","last_fullsized":"2023-10-09T06:22:40.855641+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147385+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:22:46.147385+0000","last_peered":"2023-10-09T06:22:46.147385+0000","last_clean":"2023-10-09T06:22:46.147385+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:22:46.147385+0000","last_undegraded":"2023-10-09T06:22:46.147385+0000","last_fullsized":"2023-10-09T06:22:46.147385+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855987+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:22:40.855987+0000","last_peered":"2023-10-09T06:22:40.855987+0000","last_clean":"2023-10-09T06:22:40.855987+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:22:40.855987+0000","last_undegraded":"2023-10-09T06:22:40.855987+0000","last_fullsized":"2023-10-09T06:22:40.855987+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":39,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885075+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:22:45.885075+0000","last_peered":"2023-10-09T06:22:45.885075+0000","last_clean":"2023-10-09T06:22:45.885075+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:22:45.885075+0000","last_undegraded":"2023-10-09T06:22:45.885075+0000","last_fullsized":"2023-10-09T06:22:45.885075+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":87,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.895232+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:22:44.895232+0000","last_peered":"2023-10-09T06:22:44.895232+0000","last_clean":"2023-10-09T06:22:44.895232+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:22:44.895232+0000","last_undegraded":"2023-10-09T06:22:44.895232+0000","last_fullsized":"2023-10-09T06:22:44.895232+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":151,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146858+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:22:46.146858+0000","last_peered":"2023-10-09T06:22:46.146858+0000","last_clean":"2023-10-09T06:22:46.146858+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:22:46.146858+0000","last_undegraded":"2023-10-09T06:22:46.146858+0000","last_fullsized":"2023-10-09T06:22:46.146858+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147040+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:22:46.147040+0000","last_peered":"2023-10-09T06:22:46.147040+0000","last_clean":"2023-10-09T06:22:46.147040+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:22:46.147040+0000","last_undegraded":"2023-10-09T06:22:46.147040+0000","last_fullsized":"2023-10-09T06:22:46.147040+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":39,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.884034+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:22:45.884034+0000","last_peered":"2023-10-09T06:22:45.884034+0000","last_clean":"2023-10-09T06:22:45.884034+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:22:45.884034+0000","last_undegraded":"2023-10-09T06:22:45.884034+0000","last_fullsized":"2023-10-09T06:22:45.884034+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":128,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856403+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:22:40.856403+0000","last_peered":"2023-10-09T06:22:40.856403+0000","last_clean":"2023-10-09T06:22:40.856403+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:22:40.856403+0000","last_undegraded":"2023-10-09T06:22:40.856403+0000","last_fullsized":"2023-10-09T06:22:40.856403+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877277+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:22:45.877277+0000","last_peered":"2023-10-09T06:22:45.877277+0000","last_clean":"2023-10-09T06:22:45.877277+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:22:45.877277+0000","last_undegraded":"2023-10-09T06:22:45.877277+0000","last_fullsized":"2023-10-09T06:22:45.877277+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146009+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:22:46.146009+0000","last_peered":"2023-10-09T06:22:46.146009+0000","last_clean":"2023-10-09T06:22:46.146009+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:22:46.146009+0000","last_undegraded":"2023-10-09T06:22:46.146009+0000","last_fullsized":"2023-10-09T06:22:46.146009+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":33,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.883988+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:22:45.883988+0000","last_peered":"2023-10-09T06:22:45.883988+0000","last_clean":"2023-10-09T06:22:45.883988+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:22:45.883988+0000","last_undegraded":"2023-10-09T06:22:45.883988+0000","last_fullsized":"2023-10-09T06:22:45.883988+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877422+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:22:45.877422+0000","last_peered":"2023-10-09T06:22:45.877422+0000","last_clean":"2023-10-09T06:22:45.877422+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:22:45.877422+0000","last_undegraded":"2023-10-09T06:22:45.877422+0000","last_fullsized":"2023-10-09T06:22:45.877422+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.884393+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:22:45.884393+0000","last_peered":"2023-10-09T06:22:45.884393+0000","last_clean":"2023-10-09T06:22:45.884393+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:22:45.884393+0000","last_undegraded":"2023-10-09T06:22:45.884393+0000","last_fullsized":"2023-10-09T06:22:45.884393+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":141,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856795+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:22:40.856795+0000","last_peered":"2023-10-09T06:22:40.856795+0000","last_clean":"2023-10-09T06:22:40.856795+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:22:40.856795+0000","last_undegraded":"2023-10-09T06:22:40.856795+0000","last_fullsized":"2023-10-09T06:22:40.856795+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":37,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885893+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:22:45.885893+0000","last_peered":"2023-10-09T06:22:45.885893+0000","last_clean":"2023-10-09T06:22:45.885893+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:22:45.885893+0000","last_undegraded":"2023-10-09T06:22:45.885893+0000","last_fullsized":"2023-10-09T06:22:45.885893+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.145719+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:22:46.145719+0000","last_peered":"2023-10-09T06:22:46.145719+0000","last_clean":"2023-10-09T06:22:46.145719+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:22:46.145719+0000","last_undegraded":"2023-10-09T06:22:46.145719+0000","last_fullsized":"2023-10-09T06:22:46.145719+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":37,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885805+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:22:45.885805+0000","last_peered":"2023-10-09T06:22:45.885805+0000","last_clean":"2023-10-09T06:22:45.885805+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:22:45.885805+0000","last_undegraded":"2023-10-09T06:22:45.885805+0000","last_fullsized":"2023-10-09T06:22:45.885805+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147094+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:22:46.147094+0000","last_peered":"2023-10-09T06:22:46.147094+0000","last_clean":"2023-10-09T06:22:46.147094+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:22:46.147094+0000","last_undegraded":"2023-10-09T06:22:46.147094+0000","last_fullsized":"2023-10-09T06:22:46.147094+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":37,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885405+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:22:45.885405+0000","last_peered":"2023-10-09T06:22:45.885405+0000","last_clean":"2023-10-09T06:22:45.885405+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:22:45.885405+0000","last_undegraded":"2023-10-09T06:22:45.885405+0000","last_fullsized":"2023-10-09T06:22:45.885405+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.884847+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:22:45.884847+0000","last_peered":"2023-10-09T06:22:45.884847+0000","last_clean":"2023-10-09T06:22:45.884847+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:22:45.884847+0000","last_undegraded":"2023-10-09T06:22:45.884847+0000","last_fullsized":"2023-10-09T06:22:45.884847+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146372+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:22:46.146372+0000","last_peered":"2023-10-09T06:22:46.146372+0000","last_clean":"2023-10-09T06:22:46.146372+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:22:46.146372+0000","last_undegraded":"2023-10-09T06:22:46.146372+0000","last_fullsized":"2023-10-09T06:22:46.146372+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":92,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877297+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:22:45.877297+0000","last_peered":"2023-10-09T06:22:45.877297+0000","last_clean":"2023-10-09T06:22:45.877297+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:22:45.877297+0000","last_undegraded":"2023-10-09T06:22:45.877297+0000","last_fullsized":"2023-10-09T06:22:45.877297+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":109,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856041+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:22:40.856041+0000","last_peered":"2023-10-09T06:22:40.856041+0000","last_clean":"2023-10-09T06:22:40.856041+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:22:40.856041+0000","last_undegraded":"2023-10-09T06:22:40.856041+0000","last_fullsized":"2023-10-09T06:22:40.856041+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146676+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:22:46.146676+0000","last_peered":"2023-10-09T06:22:46.146676+0000","last_clean":"2023-10-09T06:22:46.146676+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:22:46.146676+0000","last_undegraded":"2023-10-09T06:22:46.146676+0000","last_fullsized":"2023-10-09T06:22:46.146676+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885264+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:22:45.885264+0000","last_peered":"2023-10-09T06:22:45.885264+0000","last_clean":"2023-10-09T06:22:45.885264+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:22:45.885264+0000","last_undegraded":"2023-10-09T06:22:45.885264+0000","last_fullsized":"2023-10-09T06:22:45.885264+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147725+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:22:46.147725+0000","last_peered":"2023-10-09T06:22:46.147725+0000","last_clean":"2023-10-09T06:22:46.147725+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:22:46.147725+0000","last_undegraded":"2023-10-09T06:22:46.147725+0000","last_fullsized":"2023-10-09T06:22:46.147725+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146382+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:22:46.146382+0000","last_peered":"2023-10-09T06:22:46.146382+0000","last_clean":"2023-10-09T06:22:46.146382+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:22:46.146382+0000","last_undegraded":"2023-10-09T06:22:46.146382+0000","last_fullsized":"2023-10-09T06:22:46.146382+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":105,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.148076+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:22:46.148076+0000","last_peered":"2023-10-09T06:22:46.148076+0000","last_clean":"2023-10-09T06:22:46.148076+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:22:46.148076+0000","last_undegraded":"2023-10-09T06:22:46.148076+0000","last_fullsized":"2023-10-09T06:22:46.148076+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":129,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855873+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:22:40.855873+0000","last_peered":"2023-10-09T06:22:40.855873+0000","last_clean":"2023-10-09T06:22:40.855873+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:22:40.855873+0000","last_undegraded":"2023-10-09T06:22:40.855873+0000","last_fullsized":"2023-10-09T06:22:40.855873+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":88,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.898773+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:22:44.898773+0000","last_peered":"2023-10-09T06:22:44.898773+0000","last_clean":"2023-10-09T06:22:44.898773+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:22:44.898773+0000","last_undegraded":"2023-10-09T06:22:44.898773+0000","last_fullsized":"2023-10-09T06:22:44.898773+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":980,"reported_epoch":140,"state":"peering","last_fresh":"2023-10-09T06:22:45.875135+0000","last_change":"2023-10-09T06:22:45.875362+0000","last_active":"2023-10-09T06:22:45.875135+0000","last_peered":"2023-10-09T06:22:45.088273+0000","last_clean":"2023-10-09T06:18:54.772534+0000","last_became_active":"2023-10-09T06:19:00.832133+0000","last_became_peered":"2023-10-09T06:19:00.832133+0000","last_unstale":"2023-10-09T06:22:45.875135+0000","last_undegraded":"2023-10-09T06:22:45.875135+0000","last_fullsized":"2023-10-09T06:22:45.875135+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:24.389967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,7],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":121,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146205+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:22:46.146205+0000","last_peered":"2023-10-09T06:22:46.146205+0000","last_clean":"2023-10-09T06:22:46.146205+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:22:46.146205+0000","last_undegraded":"2023-10-09T06:22:46.146205+0000","last_fullsized":"2023-10-09T06:22:46.146205+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.895134+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:22:44.895134+0000","last_peered":"2023-10-09T06:22:44.895134+0000","last_clean":"2023-10-09T06:22:44.895134+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:22:44.895134+0000","last_undegraded":"2023-10-09T06:22:44.895134+0000","last_fullsized":"2023-10-09T06:22:44.895134+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":722,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146049+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:22:46.146049+0000","last_peered":"2023-10-09T06:22:46.146049+0000","last_clean":"2023-10-09T06:22:46.146049+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:22:46.146049+0000","last_undegraded":"2023-10-09T06:22:46.146049+0000","last_fullsized":"2023-10-09T06:22:46.146049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146813+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:22:46.146813+0000","last_peered":"2023-10-09T06:22:46.146813+0000","last_clean":"2023-10-09T06:22:46.146813+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:22:46.146813+0000","last_undegraded":"2023-10-09T06:22:46.146813+0000","last_fullsized":"2023-10-09T06:22:46.146813+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.899342+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:22:44.899342+0000","last_peered":"2023-10-09T06:22:44.899342+0000","last_clean":"2023-10-09T06:22:44.899342+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:22:44.899342+0000","last_undegraded":"2023-10-09T06:22:44.899342+0000","last_fullsized":"2023-10-09T06:22:44.899342+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":499,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.899274+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:22:44.899274+0000","last_peered":"2023-10-09T06:22:44.899274+0000","last_clean":"2023-10-09T06:22:44.899274+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:22:44.899274+0000","last_undegraded":"2023-10-09T06:22:44.899274+0000","last_fullsized":"2023-10-09T06:22:44.899274+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.894696+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:22:44.894696+0000","last_peered":"2023-10-09T06:22:44.894696+0000","last_clean":"2023-10-09T06:22:44.894696+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:22:44.894696+0000","last_undegraded":"2023-10-09T06:22:44.894696+0000","last_fullsized":"2023-10-09T06:22:44.894696+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.898680+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:22:44.898680+0000","last_peered":"2023-10-09T06:22:44.898680+0000","last_clean":"2023-10-09T06:22:44.898680+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:22:44.898680+0000","last_undegraded":"2023-10-09T06:22:44.898680+0000","last_fullsized":"2023-10-09T06:22:44.898680+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":489,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146851+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:22:46.146851+0000","last_peered":"2023-10-09T06:22:46.146851+0000","last_clean":"2023-10-09T06:22:46.146851+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:22:46.146851+0000","last_undegraded":"2023-10-09T06:22:46.146851+0000","last_fullsized":"2023-10-09T06:22:46.146851+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":337,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.884063+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:22:45.884063+0000","last_peered":"2023-10-09T06:22:45.884063+0000","last_clean":"2023-10-09T06:22:45.884063+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:22:45.884063+0000","last_undegraded":"2023-10-09T06:22:45.884063+0000","last_fullsized":"2023-10-09T06:22:45.884063+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856358+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:22:40.856358+0000","last_peered":"2023-10-09T06:22:40.856358+0000","last_clean":"2023-10-09T06:22:40.856358+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:22:40.856358+0000","last_undegraded":"2023-10-09T06:22:40.856358+0000","last_fullsized":"2023-10-09T06:22:40.856358+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885221+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:22:45.885221+0000","last_peered":"2023-10-09T06:22:45.885221+0000","last_clean":"2023-10-09T06:22:45.885221+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:22:45.885221+0000","last_undegraded":"2023-10-09T06:22:45.885221+0000","last_fullsized":"2023-10-09T06:22:45.885221+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":62,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.883703+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:22:45.883703+0000","last_peered":"2023-10-09T06:22:45.883703+0000","last_clean":"2023-10-09T06:22:45.883703+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:22:45.883703+0000","last_undegraded":"2023-10-09T06:22:45.883703+0000","last_fullsized":"2023-10-09T06:22:45.883703+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877033+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:22:45.877033+0000","last_peered":"2023-10-09T06:22:45.877033+0000","last_clean":"2023-10-09T06:22:45.877033+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:22:45.877033+0000","last_undegraded":"2023-10-09T06:22:45.877033+0000","last_fullsized":"2023-10-09T06:22:45.877033+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877541+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:22:45.877541+0000","last_peered":"2023-10-09T06:22:45.877541+0000","last_clean":"2023-10-09T06:22:45.877541+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:22:45.877541+0000","last_undegraded":"2023-10-09T06:22:45.877541+0000","last_fullsized":"2023-10-09T06:22:45.877541+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":51,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.885236+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:22:45.885236+0000","last_peered":"2023-10-09T06:22:45.885236+0000","last_clean":"2023-10-09T06:22:45.885236+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:22:45.885236+0000","last_undegraded":"2023-10-09T06:22:45.885236+0000","last_fullsized":"2023-10-09T06:22:45.885236+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.899264+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:22:44.899264+0000","last_peered":"2023-10-09T06:22:44.899264+0000","last_clean":"2023-10-09T06:22:44.899264+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:22:44.899264+0000","last_undegraded":"2023-10-09T06:22:44.899264+0000","last_fullsized":"2023-10-09T06:22:44.899264+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877515+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:22:45.877515+0000","last_peered":"2023-10-09T06:22:45.877515+0000","last_clean":"2023-10-09T06:22:45.877515+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:22:45.877515+0000","last_undegraded":"2023-10-09T06:22:45.877515+0000","last_fullsized":"2023-10-09T06:22:45.877515+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":135,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.857165+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:22:40.857165+0000","last_peered":"2023-10-09T06:22:40.857165+0000","last_clean":"2023-10-09T06:22:40.857165+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:22:40.857165+0000","last_undegraded":"2023-10-09T06:22:40.857165+0000","last_fullsized":"2023-10-09T06:22:40.857165+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":444,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.896470+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:22:44.896470+0000","last_peered":"2023-10-09T06:22:44.896470+0000","last_clean":"2023-10-09T06:22:44.896470+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:22:44.896470+0000","last_undegraded":"2023-10-09T06:22:44.896470+0000","last_fullsized":"2023-10-09T06:22:44.896470+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":134,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855780+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:22:40.855780+0000","last_peered":"2023-10-09T06:22:40.855780+0000","last_clean":"2023-10-09T06:22:40.855780+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:22:40.855780+0000","last_undegraded":"2023-10-09T06:22:40.855780+0000","last_fullsized":"2023-10-09T06:22:40.855780+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":837,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.890972+0000","last_change":"2023-10-09T06:22:46.890972+0000","last_active":"2023-10-09T06:22:46.890972+0000","last_peered":"2023-10-09T06:22:46.890972+0000","last_clean":"2023-10-09T06:22:46.890972+0000","last_became_active":"2023-10-09T06:22:46.889916+0000","last_became_peered":"2023-10-09T06:22:46.889916+0000","last_unstale":"2023-10-09T06:22:46.890972+0000","last_undegraded":"2023-10-09T06:22:46.890972+0000","last_fullsized":"2023-10-09T06:22:46.890972+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":96,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877482+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:22:45.877482+0000","last_peered":"2023-10-09T06:22:45.877482+0000","last_clean":"2023-10-09T06:22:45.877482+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:22:45.877482+0000","last_undegraded":"2023-10-09T06:22:45.877482+0000","last_fullsized":"2023-10-09T06:22:45.877482+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":90,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146609+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:22:46.146609+0000","last_peered":"2023-10-09T06:22:46.146609+0000","last_clean":"2023-10-09T06:22:46.146609+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:22:46.146609+0000","last_undegraded":"2023-10-09T06:22:46.146609+0000","last_fullsized":"2023-10-09T06:22:46.146609+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877468+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:22:45.877468+0000","last_peered":"2023-10-09T06:22:45.877468+0000","last_clean":"2023-10-09T06:22:45.877468+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:22:45.877468+0000","last_undegraded":"2023-10-09T06:22:45.877468+0000","last_fullsized":"2023-10-09T06:22:45.877468+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-10-09T06:22:44.903397+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:22:44.903397+0000","last_peered":"2023-10-09T06:22:44.903397+0000","last_clean":"2023-10-09T06:22:44.903397+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:22:44.903397+0000","last_undegraded":"2023-10-09T06:22:44.903397+0000","last_fullsized":"2023-10-09T06:22:44.903397+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":108,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146726+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:22:46.146726+0000","last_peered":"2023-10-09T06:22:46.146726+0000","last_clean":"2023-10-09T06:22:46.146726+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:22:46.146726+0000","last_undegraded":"2023-10-09T06:22:46.146726+0000","last_fullsized":"2023-10-09T06:22:46.146726+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147492+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:22:46.147492+0000","last_peered":"2023-10-09T06:22:46.147492+0000","last_clean":"2023-10-09T06:22:46.147492+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:22:46.147492+0000","last_undegraded":"2023-10-09T06:22:46.147492+0000","last_fullsized":"2023-10-09T06:22:46.147492+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":93,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146206+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:22:46.146206+0000","last_peered":"2023-10-09T06:22:46.146206+0000","last_clean":"2023-10-09T06:22:46.146206+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:22:46.146206+0000","last_undegraded":"2023-10-09T06:22:46.146206+0000","last_fullsized":"2023-10-09T06:22:46.146206+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.147631+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:22:46.147631+0000","last_peered":"2023-10-09T06:22:46.147631+0000","last_clean":"2023-10-09T06:22:46.147631+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:22:46.147631+0000","last_undegraded":"2023-10-09T06:22:46.147631+0000","last_fullsized":"2023-10-09T06:22:46.147631+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":117,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.856581+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:22:40.856581+0000","last_peered":"2023-10-09T06:22:40.856581+0000","last_clean":"2023-10-09T06:22:40.856581+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:22:40.856581+0000","last_undegraded":"2023-10-09T06:22:40.856581+0000","last_fullsized":"2023-10-09T06:22:40.856581+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":129,"reported_epoch":138,"state":"active+clean","last_fresh":"2023-10-09T06:22:40.855715+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:22:40.855715+0000","last_peered":"2023-10-09T06:22:40.855715+0000","last_clean":"2023-10-09T06:22:40.855715+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:22:40.855715+0000","last_undegraded":"2023-10-09T06:22:40.855715+0000","last_fullsized":"2023-10-09T06:22:40.855715+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":96,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:45.877710+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:22:45.877710+0000","last_peered":"2023-10-09T06:22:45.877710+0000","last_clean":"2023-10-09T06:22:45.877710+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:22:45.877710+0000","last_undegraded":"2023-10-09T06:22:45.877710+0000","last_fullsized":"2023-10-09T06:22:45.877710+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":105,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.146183+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:22:46.146183+0000","last_peered":"2023-10-09T06:22:46.146183+0000","last_clean":"2023-10-09T06:22:46.146183+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:22:46.146183+0000","last_undegraded":"2023-10-09T06:22:46.146183+0000","last_fullsized":"2023-10-09T06:22:46.146183+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430498,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299164,"kb_used_data":1940,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93445988,"statfs":{"total":95995035648,"available":95688691712,"internally_reserved":0,"allocated":1986560,"data_stored":1462867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.4769999999999999,"15min":1.4039999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7969999999999997,"5min":107.36799999999999,"15min":107.36799999999999},"last":0.75},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.4359999999999999,"15min":1.391},"min":{"1min":0.371,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.8480000000000008,"5min":105.89,"15min":105.89},"last":0.90500000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:21:58 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":2.7799999999999998,"15min":1.514},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1989999999999998,"5min":104.825,"15min":104.825},"last":1.133},{"interface":"front","average":{"1min":1.3640000000000001,"5min":2.3980000000000001,"15min":1.4179999999999999},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":4.351,"5min":104.637,"15min":104.637},"last":1.073}]},{"osd":2,"last update":"Mon Oct 9 06:22:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8419999999999996,"5min":4.1180000000000003,"15min":1.992},"min":{"1min":0.46700000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":61.822000000000003,"5min":104.992,"15min":104.992},"last":0.86199999999999999},{"interface":"front","average":{"1min":5.3449999999999998,"5min":4.0529999999999999,"15min":2.012},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":65.064999999999998,"5min":104.691,"15min":104.691},"last":1}]},{"osd":3,"last update":"Mon Oct 9 06:21:55 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":1.3360000000000001,"15min":1.121},"min":{"1min":0.44,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.6869999999999994,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.411,"5min":1.163,"15min":1.151},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.7519999999999998,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":0.60599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.804,"15min":1.214},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.029999999999999,"5min":11.029999999999999,"15min":11.029999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":2.754,"5min":1.9370000000000001,"15min":1.2769999999999999},"min":{"1min":0.36699999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":0.67600000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.8680000000000001,"15min":1.3009999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3339999999999996,"5min":11.996,"15min":11.996},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.837,"15min":1.3080000000000001},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.9019999999999992,"5min":13.634,"15min":13.634},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.4359999999999999,"15min":1.1990000000000001},"min":{"1min":0.40000000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":8.9260000000000002,"5min":9.9510000000000005,"15min":9.9510000000000005},"last":0.83999999999999997},{"interface":"front","average":{"1min":2.02,"5min":1.708,"15min":1.282},"min":{"1min":0.53500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.5899999999999999,"5min":8.6199999999999992,"15min":8.6199999999999992},"last":1.048}]}]},{"osd":6,"up_from":42,"seq":180388626730,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303244,"kb_used_data":1924,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441908,"statfs":{"total":95995035648,"available":95684513792,"internally_reserved":0,"allocated":1970176,"data_stored":1462110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:33 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.706,"15min":1.1639999999999999},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.1959999999999997,"5min":32.289999999999999,"15min":32.289999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.5549999999999999,"15min":1.1359999999999999},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2199999999999998,"5min":31.454999999999998,"15min":31.454999999999998},"last":1.127}]},{"osd":1,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":8.282,"5min":2.605,"15min":1.448},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":137.93899999999999,"5min":137.93899999999999,"15min":137.93899999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":7.8799999999999999,"5min":2.6429999999999998,"15min":1.476},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":138.02500000000001,"5min":138.02500000000001,"15min":138.02500000000001},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:22:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":2.9649999999999999,"15min":1.637},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.952,"5min":95.540000000000006,"15min":95.540000000000006},"last":1.0720000000000001},{"interface":"front","average":{"1min":2.1440000000000001,"5min":3.073,"15min":1.655},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":6.8250000000000002,"5min":96.352000000000004,"15min":96.352000000000004},"last":0.99099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.2030000000000001,"15min":1.1080000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.0650000000000004,"5min":7.3410000000000002,"15min":7.3410000000000002},"last":0.94199999999999995},{"interface":"front","average":{"1min":1.304,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.5220000000000002,"5min":7.5220000000000002,"15min":7.5220000000000002},"last":0.82799999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.5509999999999999,"15min":1.135},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.665,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.746},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.6679999999999999,"15min":1.212},"min":{"1min":0.35799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.300000000000001,"5min":10.300000000000001,"15min":10.300000000000001},"last":1.0169999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2480000000000002,"5min":1.802,"15min":1.2729999999999999},"min":{"1min":0.45300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.754,"5min":11.754,"15min":11.754},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.75,"15min":1.284},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.295999999999999,"5min":12.295999999999999,"15min":12.295999999999999},"last":1.153}]},{"osd":7,"last update":"Mon Oct 9 06:22:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.7709999999999999,"15min":1.3200000000000001},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.4179999999999993,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":1.0449999999999999},{"interface":"front","average":{"1min":2.5070000000000001,"5min":1.921,"15min":1.3959999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.102,"5min":11.102,"15min":11.102},"last":1.171}]}]},{"osd":1,"up_from":93,"seq":399431958639,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131604,"kb_used_data":1484,"kb_used_omap":7,"kb_used_meta":130104,"kb_avail":93613548,"statfs":{"total":95995035648,"available":95860273152,"internally_reserved":0,"allocated":1519616,"data_stored":882985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":133226896},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.427,"15min":0.95099999999999996},"min":{"1min":0.219,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":4.5679999999999996,"5min":31.969000000000001,"15min":31.969000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":2.5379999999999998,"5min":1.869,"15min":1.079},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":14.112,"5min":40.667999999999999,"15min":40.667999999999999},"last":0.77400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.319,"15min":0.89200000000000002},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.0259999999999998,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.3580000000000001,"15min":1.004},"min":{"1min":0.39600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6959999999999997,"5min":8.3960000000000008,"15min":8.3960000000000008},"last":1.105}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.27,"15min":0.97699999999999998},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.9320000000000004,"5min":16.390000000000001,"15min":16.390000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.286,"5min":1.1359999999999999,"15min":0.89200000000000002},"min":{"1min":0.28899999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.1239999999999997,"5min":16.555,"15min":16.555},"last":1.0669999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.3839999999999999,"15min":1.1140000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.0279999999999996,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.774,"5min":1.5660000000000001,"15min":1.1879999999999999},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.601,"5min":7.3639999999999999,"15min":7.3639999999999999},"last":1.1559999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.7689999999999999,"15min":1.274},"min":{"1min":0.46999999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.8049999999999997,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.91600000000000004},{"interface":"front","average":{"1min":2.6960000000000002,"5min":1.6819999999999999,"15min":1.258},"min":{"1min":0.47799999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.8860000000000001,"5min":5.6790000000000003,"15min":5.6790000000000003},"last":1.1830000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":1.3799999999999999,"15min":1.1719999999999999},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.6820000000000004,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":1.042},{"interface":"front","average":{"1min":1.966,"5min":1.4730000000000001,"15min":1.2410000000000001},"min":{"1min":0.54700000000000004,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.3609999999999998,"5min":6.3369999999999997,"15min":6.3369999999999997},"last":1.2310000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":1.8700000000000001,"15min":1.375},"min":{"1min":0.57099999999999995,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.806,"5min":7.46,"15min":7.46},"last":1.0069999999999999},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.5389999999999999,"15min":1.286},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":5.5419999999999998,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":1.5800000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122165,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":776360,"kb_used_data":610144,"kb_used_omap":8,"kb_used_meta":166199,"kb_avail":92968792,"statfs":{"total":95995035648,"available":95200043008,"internally_reserved":0,"allocated":624787456,"data_stored":624138267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":170188032},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.5600000000000001,"15min":0.92600000000000005},"min":{"1min":0.26100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.460000000000001,"5min":12.460000000000001,"15min":12.460000000000001},"last":1.093},{"interface":"front","average":{"1min":2.2370000000000001,"5min":1.4359999999999999,"15min":0.95999999999999996},"min":{"1min":0.29099999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9640000000000004,"5min":10.243,"15min":10.243},"last":1.0649999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.532,"15min":0.94999999999999996},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3570000000000002,"5min":13.035,"15min":13.035},"last":0.88200000000000001},{"interface":"front","average":{"1min":2.024,"5min":1.738,"15min":1.0640000000000001},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.006,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.92900000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.1419999999999999,"15min":0.88200000000000001},"min":{"1min":0.248,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.5490000000000004,"5min":7.5490000000000004,"15min":7.5490000000000004},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.056,"15min":0.86499999999999999},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.9610000000000003,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":1.1459999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.8320000000000001,"15min":1.228},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.8729999999999993,"5min":9.0890000000000004,"15min":9.0890000000000004},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.446,"5min":1.952,"15min":1.3120000000000001},"min":{"1min":0.48099999999999998,"5min":0.37,"15min":0.37},"max":{"1min":7.9249999999999998,"5min":10.505000000000001,"15min":10.505000000000001},"last":1.1839999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.181,"15min":1.4119999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.5339999999999998,"5min":9.548,"15min":9.548},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.7729999999999999,"5min":1.8100000000000001,"15min":1.302},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.329000000000001,"5min":10.329000000000001,"15min":10.329000000000001},"last":1.5860000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.5189999999999999,"15min":1.218},"min":{"1min":0.45500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.9169999999999998,"5min":10.194000000000001,"15min":10.194000000000001},"last":1.629},{"interface":"front","average":{"1min":1.411,"5min":1.554,"15min":1.1910000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.7629999999999999,"5min":8.7629999999999999,"15min":8.7629999999999999},"last":1.496}]},{"osd":7,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.6719999999999999,"15min":1.321},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.0330000000000004,"5min":6.0330000000000004,"15min":6.0330000000000004},"last":1.5309999999999999},{"interface":"front","average":{"1min":2.1899999999999999,"5min":2.0059999999999998,"15min":1.409},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.7959999999999994,"5min":8.7959999999999994,"15min":8.7959999999999994},"last":1.359}]}]},{"osd":2,"up_from":98,"seq":420906795113,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131524,"kb_used_data":5564,"kb_used_omap":11,"kb_used_meta":125940,"kb_avail":93613628,"statfs":{"total":95995035648,"available":95860355072,"internally_reserved":0,"allocated":5697536,"data_stored":5065450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":128963518},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":2.0150000000000001,"15min":1.135},"min":{"1min":0.224,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.0839999999999996,"5min":35.015999999999998,"15min":35.015999999999998},"last":8.609},{"interface":"front","average":{"1min":1.306,"5min":2.3719999999999999,"15min":1.2290000000000001},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":4.492,"5min":36.415999999999997,"15min":36.415999999999997},"last":6.7469999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.29699999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.6050000000000004,"5min":8.298,"15min":8.298},"last":8.5399999999999991},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.9099999999999999,"15min":1.113},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.0350000000000001,"5min":10.4,"15min":10.4},"last":1.1240000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:21:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.2110000000000001,"15min":0.89000000000000001},"min":{"1min":0.30099999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.2650000000000006,"5min":8.2650000000000006,"15min":8.2650000000000006},"last":6.7279999999999998},{"interface":"front","average":{"1min":2.2610000000000001,"5min":1.167,"15min":0.85299999999999998},"min":{"1min":0.312,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":18.831,"5min":18.831,"15min":18.831},"last":2.5899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":2.5899999999999999,"15min":1.502},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":5.7439999999999998,"5min":19.184000000000001,"15min":19.184000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.2090000000000001,"15min":1.363},"min":{"1min":0.41699999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.408000000000001,"5min":29.408000000000001,"15min":29.408000000000001},"last":2.665}]},{"osd":5,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.2320000000000002,"15min":1.417},"min":{"1min":0.39700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":8.0830000000000002,"5min":12.833,"15min":12.833},"last":1.2689999999999999},{"interface":"front","average":{"1min":3.0449999999999999,"5min":1.9770000000000001,"15min":1.2949999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":29.545000000000002,"5min":29.545000000000002,"15min":29.545000000000002},"last":2.71}]},{"osd":6,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":1.5569999999999999,"15min":1.1910000000000001},"min":{"1min":0.374,"5min":0.372,"15min":0.372},"max":{"1min":29.504000000000001,"5min":29.504000000000001,"15min":29.504000000000001},"last":2.7469999999999999},{"interface":"front","average":{"1min":1.986,"5min":1.698,"15min":1.234},"min":{"1min":0.46899999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.6760000000000002,"5min":8.1349999999999998,"15min":8.1349999999999998},"last":6.6779999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.827,"15min":1.3380000000000001},"min":{"1min":0.52200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8630000000000004,"5min":8.3309999999999995,"15min":8.3309999999999995},"last":6.601},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.2799999999999998,"15min":1.494},"min":{"1min":0.435,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":4.585,"5min":10.978999999999999,"15min":10.978999999999999},"last":1.204}]}]},{"osd":3,"up_from":103,"seq":442381631587,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77236,"kb_used_data":5548,"kb_used_omap":3,"kb_used_meta":71676,"kb_avail":93667916,"statfs":{"total":95995035648,"available":95915945984,"internally_reserved":0,"allocated":5681152,"data_stored":5063660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":73396769},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":1.996,"15min":1.111},"min":{"1min":0.25,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":9.5739999999999998,"5min":62.939,"15min":62.939},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.5510000000000002,"5min":2.0819999999999999,"15min":1.125},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":7.2789999999999999,"5min":63.869,"15min":63.869},"last":0.42199999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":0.88600000000000001,"15min":0.76800000000000002},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.944,"5min":5.944,"15min":5.944},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.335,"5min":1.0600000000000001,"15min":0.86299999999999999},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.94599999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":2.2570000000000001,"15min":1.2989999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.891999999999999,"5min":70.594999999999999,"15min":70.594999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.819,"15min":1.095},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.8890000000000002,"5min":62.991999999999997,"15min":62.991999999999997},"last":0.64000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4460000000000002,"5min":1.7529999999999999,"15min":1.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":10.446999999999999,"5min":10.446999999999999,"15min":10.446999999999999},"last":1.012},{"interface":"front","average":{"1min":2.5190000000000001,"5min":1.5309999999999999,"15min":1.177},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":7.8940000000000001,"5min":7.8940000000000001,"15min":7.8940000000000001},"last":1.236}]},{"osd":5,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.3300000000000001,"15min":1.1359999999999999},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.9870000000000001,"5min":6.0910000000000002,"15min":6.0910000000000002},"last":1.089},{"interface":"front","average":{"1min":3.2389999999999999,"5min":1.758,"15min":1.304},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":11.454000000000001,"5min":11.754,"15min":11.754},"last":1.3280000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.1160000000000001,"15min":1.119},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":1.3089999999999999},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.2809999999999999,"15min":1.159},"min":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.0250000000000004,"5min":6.0250000000000004,"15min":6.0250000000000004},"last":1.3520000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.3799999999999999,"15min":1.1679999999999999},"min":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.4820000000000002,"5min":6.4820000000000002,"15min":6.4820000000000002},"last":1.1439999999999999},{"interface":"front","average":{"1min":2.3250000000000002,"5min":1.4630000000000001,"15min":1.2470000000000001},"min":{"1min":0.42099999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":11.439,"5min":11.439,"15min":11.439},"last":1.1870000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019196,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":596884,"kb_used_data":287372,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93148268,"statfs":{"total":95995035648,"available":95383826432,"internally_reserved":0,"allocated":294268928,"data_stored":293752813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.6240000000000001,"15min":1.859},"min":{"1min":0.32500000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.3949999999999996,"5min":84.888000000000005,"15min":84.888000000000005},"last":1.1519999999999999},{"interface":"front","average":{"1min":2.5950000000000002,"5min":3.9300000000000002,"15min":1.994},"min":{"1min":0.39500000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8049999999999997,"5min":84.912999999999997,"15min":84.912999999999997},"last":1.21}]},{"osd":1,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5270000000000001,"5min":2.383,"15min":1.4350000000000001},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":61.726999999999997,"5min":61.726999999999997,"15min":61.726999999999997},"last":1.069},{"interface":"front","average":{"1min":5.077,"5min":2.9529999999999998,"15min":1.6399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":61.932000000000002,"5min":61.932000000000002,"15min":61.932000000000002},"last":1.0389999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":3.3559999999999999,"15min":1.726},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.9489999999999998,"5min":61.783999999999999,"15min":61.783999999999999},"last":1.103},{"interface":"front","average":{"1min":2.9350000000000001,"5min":3.5550000000000002,"15min":1.7789999999999999},"min":{"1min":0.36699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.56,"5min":61.551000000000002,"15min":61.551000000000002},"last":1.2609999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":2.2080000000000002,"15min":1.419},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.217000000000001,"5min":36.154000000000003,"15min":36.154000000000003},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.254,"5min":1.6479999999999999,"15min":1.236},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.4550000000000001,"5min":36.064,"15min":36.064},"last":1.296}]},{"osd":5,"last update":"Mon Oct 9 06:22:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":1.859,"15min":1.224},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.316},"max":{"1min":12.242000000000001,"5min":12.242000000000001,"15min":12.242000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":3.2080000000000002,"5min":2.5720000000000001,"15min":1.5069999999999999},"min":{"1min":0.28499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.234999999999999,"5min":20.113,"15min":20.113},"last":0.75800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.1749999999999998,"15min":1.413},"min":{"1min":0.374,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.8529999999999998,"5min":18.010000000000002,"15min":18.010000000000002},"last":0.995},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.677,"15min":1.2410000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.867,"5min":13.913,"15min":13.913},"last":1.238}]},{"osd":7,"last update":"Mon Oct 9 06:22:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.2850000000000001,"15min":1.478},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":7.8840000000000003,"5min":15.294,"15min":15.294},"last":1.3140000000000001},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.3679999999999999,"15min":1.5009999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.731999999999999,"5min":10.731999999999999,"15min":10.731999999999999},"last":0.91200000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822963,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302656,"kb_used_data":1336,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442496,"statfs":{"total":95995035648,"available":95685115904,"internally_reserved":0,"allocated":1368064,"data_stored":867562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":3.5710000000000002,"15min":1.8149999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":5.6719999999999997,"5min":69.793999999999997,"15min":69.793999999999997},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.774,"5min":2.9969999999999999,"15min":1.5980000000000001},"min":{"1min":0.35599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.0090000000000003,"5min":68.873000000000005,"15min":68.873000000000005},"last":1.0740000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:10 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":2.4460000000000002,"15min":1.4399999999999999},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":23.396000000000001,"5min":52.070999999999998,"15min":52.070999999999998},"last":0.84599999999999997},{"interface":"front","average":{"1min":3.5990000000000002,"5min":2.7749999999999999,"15min":1.5269999999999999},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":22.032,"5min":51.570999999999998,"15min":51.570999999999998},"last":0.89600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":4.5620000000000003,"5min":3.1120000000000001,"15min":1.657},"min":{"1min":0.313,"5min":0.307,"15min":0.307},"max":{"1min":45.466000000000001,"5min":47.207000000000001,"15min":47.207000000000001},"last":1.1819999999999999},{"interface":"front","average":{"1min":4.6390000000000002,"5min":3.1419999999999999,"15min":1.728},"min":{"1min":0.39000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":59.082999999999998,"5min":59.082999999999998,"15min":59.082999999999998},"last":1.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:21:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.746,"15min":1.256},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.0649999999999995,"5min":12.334,"15min":12.334},"last":0.81200000000000006},{"interface":"front","average":{"1min":1.6140000000000001,"5min":1.5800000000000001,"15min":1.218},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.2069999999999999,"5min":11.823,"15min":11.823},"last":0.94399999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:22:10 2023","interfaces":[{"interface":"back","average":{"1min":3.0899999999999999,"5min":2.6819999999999999,"15min":1.494},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.617000000000001,"5min":17.667999999999999,"15min":17.667999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":2.4289999999999998,"5min":2.141,"15min":1.3839999999999999},"min":{"1min":0.44700000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.181999999999999,"5min":16.181999999999999,"15min":16.181999999999999},"last":1.0429999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:21:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":2.0870000000000002,"15min":1.3460000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.7279999999999998,"5min":17.454999999999998,"15min":17.454999999999998},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.75,"5min":1.9890000000000001,"15min":1.3859999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":12.365,"5min":12.365,"15min":12.365},"last":0.97099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.3540000000000001,"15min":1.5289999999999999},"min":{"1min":0.495,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.3170000000000002,"5min":13.047000000000001,"15min":13.047000000000001},"last":1.141},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.2130000000000001,"15min":1.4870000000000001},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":10.465999999999999,"5min":11.981,"15min":11.981},"last":1.1160000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:22:55.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:22:55.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:55.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:55.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:56.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:56.066 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:55 smithi125 bash[77299]: cluster 2023-10-09T06:22:54.778089+0000 mgr.y (mgr.25723) 531 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:56.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:56 smithi125 bash[80002]: cluster 2023-10-09T06:22:54.778089+0000 mgr.y (mgr.25723) 531 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:56.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:56.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:56.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:56.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:56.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:56.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:55 smithi155 bash[59495]: cluster 2023-10-09T06:22:54.778089+0000 mgr.y (mgr.25723) 531 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:22:56.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:56.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:56.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:56.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:56.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:56.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:56.823 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:56.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:56.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:56.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:56.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:56.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:57.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:57.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:57.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:57.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:57.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:57.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:57.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:57.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:57.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:57.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:57.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:57.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:57.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:57.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:57.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:58.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.021 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:57 smithi125 bash[80002]: cluster 2023-10-09T06:22:56.778704+0000 mgr.y (mgr.25723) 532 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:58.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:57 smithi125 bash[77299]: cluster 2023-10-09T06:22:56.778704+0000 mgr.y (mgr.25723) 532 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:58.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:58.090 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:57 smithi155 bash[59495]: cluster 2023-10-09T06:22:56.778704+0000 mgr.y (mgr.25723) 532 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:58.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:58.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:58.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:58.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.424 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:58.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:58.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:58.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:58.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:58.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:58.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:58.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:58.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:58.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:59.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:59.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:59.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:59.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:59.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:59 smithi155 bash[59495]: audit 2023-10-09T06:22:58.182441+0000 mon.a (mon.0) 732 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:59 smithi155 bash[59495]: audit 2023-10-09T06:22:58.188256+0000 mon.a (mon.0) 733 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:59 smithi155 bash[59495]: cluster 2023-10-09T06:22:58.779114+0000 mgr.y (mgr.25723) 533 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:59.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:22:59 smithi155 bash[59495]: audit 2023-10-09T06:22:58.840009+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:59.475 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:59.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:59.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:59.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[77299]: audit 2023-10-09T06:22:58.182441+0000 mon.a (mon.0) 732 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[77299]: audit 2023-10-09T06:22:58.188256+0000 mon.a (mon.0) 733 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[77299]: cluster 2023-10-09T06:22:58.779114+0000 mgr.y (mgr.25723) 533 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[77299]: audit 2023-10-09T06:22:58.840009+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[80002]: audit 2023-10-09T06:22:58.182441+0000 mon.a (mon.0) 732 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[80002]: audit 2023-10-09T06:22:58.188256+0000 mon.a (mon.0) 733 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[80002]: cluster 2023-10-09T06:22:58.779114+0000 mgr.y (mgr.25723) 533 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:22:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:22:59 smithi125 bash[80002]: audit 2023-10-09T06:22:58.840009+0000 mon.a (mon.0) 734 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:22:59.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:59.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:59.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:59.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:59.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:22:59.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:59.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:00.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:00.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:00.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:00.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:00.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:00.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:00.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:00.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:00.529 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:23:00.529 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:23:00.530 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T06:23:00.530 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:23:00.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:00.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:00.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:00.644 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:23:00.666 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:23:00.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:00.689 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:23:00.713 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:23:00.741 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:23:00.773 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:23:00.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:00.802 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:23:00.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:00.830 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.832 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:23:00.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:00 smithi125 bash[80002]: audit 2023-10-09T06:23:00.614801+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.125:0/2852171860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.857 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:00 smithi125 bash[77299]: audit 2023-10-09T06:23:00.614801+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.125:0/2852171860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:00 smithi155 bash[59495]: audit 2023-10-09T06:23:00.614801+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.125:0/2852171860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:00.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:00.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:01.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:01.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:01.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:01.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:01.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:01.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:01.336 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:01] "GET /metrics HTTP/1.1" 200 34182 "" "Prometheus/2.33.4" 2023-10-09T06:23:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:01.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:01.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:23:01.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:01.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:01.609 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:01.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:01.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:01.807 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:23:01.808 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_1' created 2023-10-09T06:23:01.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:01.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:01.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:01.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:00.777801+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: cluster 2023-10-09T06:23:00.779627+0000 mgr.y (mgr.25723) 534 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:00.784955+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:00.882704+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.125:0/1800967139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:00.883132+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:01.249060+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.125:0/436527551' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:01.985 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:01.249598+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:01.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:01.693732+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:01.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:01.695319+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:01.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:01 smithi155 bash[59495]: audit 2023-10-09T06:23:01.703258+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:02.064 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:00.777801+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: cluster 2023-10-09T06:23:00.779627+0000 mgr.y (mgr.25723) 534 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:00.784955+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:00.882704+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.125:0/1800967139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:00.883132+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:01.249060+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.125:0/436527551' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:02.065 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:01.249598+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:02.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:01.693732+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:02.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:01.695319+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:02.066 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[80002]: audit 2023-10-09T06:23:01.703258+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.066 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:00.777801+0000 mon.a (mon.0) 735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.066 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: cluster 2023-10-09T06:23:00.779627+0000 mgr.y (mgr.25723) 534 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:00.784955+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:00.882704+0000 mon.c (mon.2) 53 : audit 1 from='client.? 172.21.15.125:0/1800967139' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:00.883132+0000 mon.a (mon.0) 737 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:01.249060+0000 mon.c (mon.2) 54 : audit 1 from='client.? 172.21.15.125:0/436527551' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:01.249598+0000 mon.a (mon.0) 738 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:01.693732+0000 mon.a (mon.0) 739 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:01.695319+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:02.067 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:01 smithi125 bash[77299]: audit 2023-10-09T06:23:01.703258+0000 mon.a (mon.0) 741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:02.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:02.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:02.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:02.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:02.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:02.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:02.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.679 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:02.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:02.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:02.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:02.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:02.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:03.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.782306+0000 mon.a (mon.0) 742 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: audit 2023-10-09T06:23:01.793323+0000 mon.a (mon.0) 743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: audit 2023-10-09T06:23:01.793433+0000 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.793560+0000 mon.a (mon.0) 745 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.797720+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.797916+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.798037+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.798072+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.074 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.800198+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.075 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:02 smithi155 bash[59495]: cluster 2023-10-09T06:23:01.800534+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.094 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.782306+0000 mon.a (mon.0) 742 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: audit 2023-10-09T06:23:01.793323+0000 mon.a (mon.0) 743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: audit 2023-10-09T06:23:01.793433+0000 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.793560+0000 mon.a (mon.0) 745 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.797720+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.797916+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.798037+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.798072+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.800198+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[77299]: cluster 2023-10-09T06:23:01.800534+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.782306+0000 mon.a (mon.0) 742 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: audit 2023-10-09T06:23:01.793323+0000 mon.a (mon.0) 743 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: audit 2023-10-09T06:23:01.793433+0000 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.793560+0000 mon.a (mon.0) 745 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.797720+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.797916+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.798037+0000 osd.4 (osd.4) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.798072+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.800198+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:02 smithi125 bash[80002]: cluster 2023-10-09T06:23:01.800534+0000 osd.4 (osd.4) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-09T06:23:03.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:03.229 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:03.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T06:23:03.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:03.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:03.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:03.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:03.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:03.623 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:03.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:03.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:03.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:03.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:23:03.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:03.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:03.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:03.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:03.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:04.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:04.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:04.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.780010+0000 mgr.y (mgr.25723) 535 : cluster 0 pgmap v503: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.799436+0000 mon.a (mon.0) 746 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.802628+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.803284+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.805115+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.806174+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: cluster 2023-10-09T06:23:02.806258+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: audit 2023-10-09T06:23:03.577039+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:04.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:03 smithi155 bash[59495]: audit 2023-10-09T06:23:03.578776+0000 mon.a (mon.0) 748 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:04.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:04.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:04.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:04.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:04.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.780010+0000 mgr.y (mgr.25723) 535 : cluster 0 pgmap v503: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:04.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.799436+0000 mon.a (mon.0) 746 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.802628+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.803284+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.805115+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.806174+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: cluster 2023-10-09T06:23:02.806258+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: audit 2023-10-09T06:23:03.577039+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[77299]: audit 2023-10-09T06:23:03.578776+0000 mon.a (mon.0) 748 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.780010+0000 mgr.y (mgr.25723) 535 : cluster 0 pgmap v503: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:04.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.799436+0000 mon.a (mon.0) 746 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.802628+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.803284+0000 osd.4 (osd.4) 71 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.805115+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.806174+0000 osd.4 (osd.4) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: cluster 2023-10-09T06:23:02.806258+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e143 with expected crc 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: audit 2023-10-09T06:23:03.577039+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:04.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:03 smithi125 bash[80002]: audit 2023-10-09T06:23:03.578776+0000 mon.a (mon.0) 748 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:04.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:04.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:04.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:04.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:04.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:04.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:04.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:04.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:04.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:04.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:03.810322+0000 mon.a (mon.0) 749 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:23:04.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.068887+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:04.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.071743+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:04.874 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.202303+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:04.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.202488+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:04.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:04 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.202636+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:04.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:04.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:04.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:04.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:03.810322+0000 mon.a (mon.0) 749 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.068887+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.071743+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.202303+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.202488+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.008 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.202636+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:03.810322+0000 mon.a (mon.0) 749 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.068887+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.071743+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.202303+0000 osd.4 (osd.4) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.202488+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.009 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:04 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.202636+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e144 with expected crc 2023-10-09T06:23:05.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:05.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:05.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:05.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:05.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:23:05.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:05.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:05.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:05.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:05.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:05.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:05.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:05.817 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:05.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:05.933 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:05 smithi125 bash[80002]: cluster 2023-10-09T06:23:04.780672+0000 mgr.y (mgr.25723) 536 : cluster 0 pgmap v506: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:05.933 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:05 smithi125 bash[77299]: cluster 2023-10-09T06:23:04.780672+0000 mgr.y (mgr.25723) 536 : cluster 0 pgmap v506: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:05.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:05.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:06.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:06.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:06.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:05 smithi155 bash[59495]: cluster 2023-10-09T06:23:04.780672+0000 mgr.y (mgr.25723) 536 : cluster 0 pgmap v506: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:06.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:06.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:06.247 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:06.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:06.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:06.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:06.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:06.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:06.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:06.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:06.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:06.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:06.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:06.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:06.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:06.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:07.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:07.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:07.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:07.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:07.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:07.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:07.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:07.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:07.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:07.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:07.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:07.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:07.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:07.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:07.840 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:23:07.840 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":701,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":144,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832081,"num_in_osds":8,"osd_in_since":1696832338,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":28,"data_bytes":4790195,"bytes_used":2597519360,"bytes_avail":765362765824,"bytes_total":767960285184},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.yetyhy","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T06:15:42.606466+0000","services":{}},"progress_events":{"099f191d-326e-4e30-a011-792152843ed5":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"74b9a7a6-3b80-432e-a0ed-33973b35b600":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 7m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T06:23:07.894 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:07 smithi125 bash[77299]: cluster 2023-10-09T06:23:06.781841+0000 mgr.y (mgr.25723) 537 : cluster 0 pgmap v507: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:07.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:07 smithi125 bash[80002]: cluster 2023-10-09T06:23:06.781841+0000 mgr.y (mgr.25723) 537 : cluster 0 pgmap v507: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:07.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:07.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:07.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:07.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:08.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:08.038 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:07 smithi155 bash[59495]: cluster 2023-10-09T06:23:06.781841+0000 mgr.y (mgr.25723) 537 : cluster 0 pgmap v507: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:08.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:08.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:08.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:08.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:08.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:08.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:08.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:08.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:08.741 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T06:23:08.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:23:08.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:08.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:08.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:08 smithi155 bash[59495]: audit 2023-10-09T06:23:07.839480+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.125:0/257803158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:23:08.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:08.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:08.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:09.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:09.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:09.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:08 smithi125 bash[77299]: audit 2023-10-09T06:23:07.839480+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.125:0/257803158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:23:09.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:08 smithi125 bash[80002]: audit 2023-10-09T06:23:07.839480+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.125:0/257803158' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:23:09.165 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:09.166 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:09.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:09.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:09.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:09.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:09.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:09.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:09.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:09.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:09.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:09.646 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:09.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:09.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:09.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:09.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:09 smithi125 bash[80002]: cluster 2023-10-09T06:23:08.782650+0000 mgr.y (mgr.25723) 538 : cluster 0 pgmap v508: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:09.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:09.885 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:23:09.885 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:23:09.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:09.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:09.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:09 smithi155 bash[59495]: cluster 2023-10-09T06:23:08.782650+0000 mgr.y (mgr.25723) 538 : cluster 0 pgmap v508: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:10.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:10.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:09 smithi125 bash[77299]: cluster 2023-10-09T06:23:08.782650+0000 mgr.y (mgr.25723) 538 : cluster 0 pgmap v508: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:10.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:10.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:10.224 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:10.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:10.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:10.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:10.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:10.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:10.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:10.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:10.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:10.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:23:10.809 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:10.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:10.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:10.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:10.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:10.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:11.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:11.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:11.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:11.225 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:11.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:11.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:11.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:11.613 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:11] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-09T06:23:11.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:11.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:11.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:11.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:11 smithi125 bash[77299]: cluster 2023-10-09T06:23:10.783585+0000 mgr.y (mgr.25723) 539 : cluster 0 pgmap v509: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:11.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:11 smithi125 bash[80002]: cluster 2023-10-09T06:23:10.783585+0000 mgr.y (mgr.25723) 539 : cluster 0 pgmap v509: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:11.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:11.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:12.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:12.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:11 smithi155 bash[59495]: cluster 2023-10-09T06:23:10.783585+0000 mgr.y (mgr.25723) 539 : cluster 0 pgmap v509: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:12.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:12.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:12.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:12.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:12.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:12.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:12.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:12.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:12.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:12.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.804 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:12.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:12.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:12.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:13.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:13.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:13.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:13.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:13.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:13.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:13.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:13.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:13.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:13.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:13.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:13.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:13.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:13.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:13.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:13 smithi125 bash[77299]: cluster 2023-10-09T06:23:12.784145+0000 mgr.y (mgr.25723) 540 : cluster 0 pgmap v510: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:13.985 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:13 smithi125 bash[80002]: cluster 2023-10-09T06:23:12.784145+0000 mgr.y (mgr.25723) 540 : cluster 0 pgmap v510: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:13.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:14.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:14.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:14.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:13 smithi155 bash[59495]: cluster 2023-10-09T06:23:12.784145+0000 mgr.y (mgr.25723) 540 : cluster 0 pgmap v510: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:14.230 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:14.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:14.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:14.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:14.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:14.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:14.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:14.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:14.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:14.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:14.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:15.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.060 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:15.142 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:14 smithi125 bash[77299]: audit 2023-10-09T06:23:13.854641+0000 mon.a (mon.0) 750 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:15.224 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:14 smithi125 bash[80002]: audit 2023-10-09T06:23:13.854641+0000 mon.a (mon.0) 750 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:15.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:14 smithi155 bash[59495]: audit 2023-10-09T06:23:13.854641+0000 mon.a (mon.0) 750 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:15.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:15.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:15.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:15.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:15.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:15.534 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:15.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:15.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:15.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:15.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:15.852 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:15.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:15 smithi155 bash[59495]: cluster 2023-10-09T06:23:14.784867+0000 mgr.y (mgr.25723) 541 : cluster 0 pgmap v511: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:15.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:15.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:16.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:16.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:16.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:15 smithi125 bash[77299]: cluster 2023-10-09T06:23:14.784867+0000 mgr.y (mgr.25723) 541 : cluster 0 pgmap v511: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:15 smithi125 bash[80002]: cluster 2023-10-09T06:23:14.784867+0000 mgr.y (mgr.25723) 541 : cluster 0 pgmap v511: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:16.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:16.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:16.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:16.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:16.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:16.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:16.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:16.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:16.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:16.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:16.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:16.828 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:16.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:16.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:16.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:17.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:17.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:17.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:17.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:17.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:17.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:17.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:17.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:17.474 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:17.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:17.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:17.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:17.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:17.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:17.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:17.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:17 smithi125 bash[77299]: cluster 2023-10-09T06:23:16.785923+0000 mgr.y (mgr.25723) 542 : cluster 0 pgmap v512: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:17.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:17 smithi125 bash[77299]: audit 2023-10-09T06:23:17.844530+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.125:0/3486311039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:17.875 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:23:17.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:17.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:17.898 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:23:17.922 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:23:17.944 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:23:17.967 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:23:17.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:17.991 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:23:18.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:18.018 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:23:18.037 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:23:18.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.043 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.044 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:23:18.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:18.123 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:17 smithi125 bash[80002]: cluster 2023-10-09T06:23:16.785923+0000 mgr.y (mgr.25723) 542 : cluster 0 pgmap v512: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:18.123 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:17 smithi125 bash[80002]: audit 2023-10-09T06:23:17.844530+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.125:0/3486311039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:18.131 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:17 smithi155 bash[59495]: cluster 2023-10-09T06:23:16.785923+0000 mgr.y (mgr.25723) 542 : cluster 0 pgmap v512: 89 pgs: 89 active+clean; 4.6 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:18.131 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:17 smithi155 bash[59495]: audit 2023-10-09T06:23:17.844530+0000 mon.a (mon.0) 751 : audit 0 from='client.? 172.21.15.125:0/3486311039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:18.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:18.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:18.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:18.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:18.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:18.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:18.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:18.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:18.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:18.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:18.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:18.887 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:23:18.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:18.908 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[77299]: audit 2023-10-09T06:23:18.491932+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.125:0/4005272224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:18.909 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[77299]: audit 2023-10-09T06:23:18.492355+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:18.909 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[77299]: audit 2023-10-09T06:23:18.571443+0000 mon.a (mon.0) 753 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:18.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[80002]: audit 2023-10-09T06:23:18.491932+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.125:0/4005272224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:18.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[80002]: audit 2023-10-09T06:23:18.492355+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:18.909 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:18 smithi125 bash[80002]: audit 2023-10-09T06:23:18.571443+0000 mon.a (mon.0) 753 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:18.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:18.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:19.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:19.043 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:18 smithi155 bash[59495]: audit 2023-10-09T06:23:18.491932+0000 mon.c (mon.2) 55 : audit 1 from='client.? 172.21.15.125:0/4005272224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:19.043 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:18 smithi155 bash[59495]: audit 2023-10-09T06:23:18.492355+0000 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:19.043 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:18 smithi155 bash[59495]: audit 2023-10-09T06:23:18.571443+0000 mon.a (mon.0) 753 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:19.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:19.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:19.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:19.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:19.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:19.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:19.476 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:23:19.476 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:23:19.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:23:19.476 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:23:19.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76675 5.9 5.5b63a009 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.805597+0000", 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.667081666, 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98876737999999997, 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76675 2023-10-09T06:23:19.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805597+0000", 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805597+0000", 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:19.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805596+0000", 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018581000000000001 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805782+0000", 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5221e-05 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805797+0000", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4739999999999999e-06 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805804+0000", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2577000000000007e-05 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805877+0000", 2023-10-09T06:23:19.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6640999999999999e-05 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.805894+0000", 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98844578599999999 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.794339+0000", 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5249000000000002e-05 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.794365+0000", 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98876737999999997 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76676 5.4 5.99f0d814 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.482 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.806138+0000", 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.666540591, 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98926034299999999, 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76676 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806138+0000", 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806138+0000", 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806138+0000", 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5807000000000003e-05 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806193+0000", 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2138e-05 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806205+0000", 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.498e-06 2023-10-09T06:23:19.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806213+0000", 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9561000000000002e-05 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806263+0000", 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8920000000000003e-06 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.806269+0000", 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98911231499999996 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795382+0000", 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6982000000000001e-05 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795399+0000", 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98926034299999999 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76683 5.9 5.b9e825e9 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.821534+0000", 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.65114462700001, 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97288229599999998, 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.488 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76683 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821534+0000", 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821534+0000", 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821532+0000", 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5476999999999995e-05 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821617+0000", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8442000000000001e-05 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821636+0000", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0923e-05 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821647+0000", 2023-10-09T06:23:19.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2400999999999998e-05 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821739+0000", 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2478e-05 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821752+0000", 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97265706500000004 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.794409+0000", 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8690000000000004e-06 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.794417+0000", 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97288229599999998 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.493 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76684 5.4 5.13376f44 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.821705+0000", 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.65097446200002, 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97373211100000001, 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.494 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76684 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821705+0000", 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821705+0000", 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:19.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821703+0000", 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3713000000000004e-05 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821777+0000", 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4032000000000001e-05 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821791+0000", 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0016e-05 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821801+0000", 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7808999999999999e-05 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821889+0000", 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3859999999999999e-05 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.821903+0000", 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97352582300000001 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795429+0000", 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0800000000000006e-06 2023-10-09T06:23:19.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795437+0000", 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97373211100000001 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76686 5.9 5.bb4b4869 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.499 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.828249+0000", 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.64442960899999, 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96892746500000004, 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76686 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828249+0000", 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828249+0000", 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828248+0000", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000130462 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828378+0000", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2293999999999999e-05 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828410+0000", 2023-10-09T06:23:19.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1070999999999999e-05 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828431+0000", 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7627999999999995e-05 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828519+0000", 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0908e-05 2023-10-09T06:23:19.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.828530+0000", 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96863589100000003 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.797166+0000", 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0917000000000001e-05 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.797177+0000", 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96892746500000004 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76687 5.4 5.47534 (undecoded) ondisk+write+known_if_redirected e117)", 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.830632+0000", 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 260.64204715599999, 2023-10-09T06:23:19.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96524010800000004, 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76687 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830632+0000", 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830632+0000", 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830630+0000", 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8612e-05 2023-10-09T06:23:19.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830699+0000", 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6030999999999999e-05 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830715+0000", 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4199999999999996e-06 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830724+0000", 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6866000000000002e-05 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830771+0000", 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5250000000000002e-06 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:23:19.509 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.830778+0000", 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96508413199999998 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795862+0000", 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0267e-05 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.795872+0000", 2023-10-09T06:23:19.510 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.96524010800000004 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 121/118 [PushOp(5:86f039a3:::benchmark_data_smithi125_106927_object5145:head, version: 112'288, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:86f039a3:::benchmark_data_smithi125_106927_object5145:head@112'288, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:18.694967+0000", 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 240.77771219600001, 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26393931700000001, 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.511 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.694967+0000", 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.694967+0000", 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.694966+0000", 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3013000000000003e-05 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.695009+0000", 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0068000000000002e-05 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.695029+0000", 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8369999999999992e-06 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.695039+0000", 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.248039281 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.943078+0000", 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7800000000000006e-06 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.943087+0000", 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015819079999999999 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.958906+0000", 2023-10-09T06:23:19.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26393931700000001 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:90023 5.1 e121/118 5:8fe4d95c:::benchmark_data_smithi125_106927_object90022:head v 121'5591, mlcod=112'286)", 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:18.745314+0000", 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 240.72736474600001, 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105867276, 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.516 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745314+0000", 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745314+0000", 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745312+0000", 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2864999999999998e-05 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745345+0000", 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.666e-05 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745362+0000", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5879999999999998e-06 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.745370+0000", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.053760781000000001 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.799131+0000", 2023-10-09T06:23:19.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011044699999999999 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.799241+0000", 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.051781494999999997 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.851023+0000", 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000158696 2023-10-09T06:23:19.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.851182+0000", 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105867276 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:90031 5.1 e121/118 5:8977b381:::benchmark_data_smithi125_106927_object90030:head v 121'5592, mlcod=112'286)", 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:18.754024+0000", 2023-10-09T06:23:19.521 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 240.71865526900001, 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101127024, 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754024+0000", 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754024+0000", 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754023+0000", 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5437000000000001e-05 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754048+0000", 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2841e-05 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754061+0000", 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6089999999999997e-06 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.754068+0000", 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.052413448000000001 2023-10-09T06:23:19.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.806481+0000", 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7886000000000003e-05 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.806519+0000", 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044666631999999998 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.851186+0000", 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039652130000000004 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:18.855151+0000", 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101127024 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.526 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 121/118 [PushOp(5:80cd118f:::benchmark_data_smithi125_106927_object5369:head, version: 112'298, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:80cd118f:::benchmark_data_smithi125_106927_object5369:head@112'298, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:19.589434+0000", 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 239.88324479299999, 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13323011100000001, 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589434+0000", 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589434+0000", 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589433+0000", 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3985e-05 2023-10-09T06:23:19.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589477+0000", 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013065799999999999 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589608+0000", 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5705e-05 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.589623+0000", 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13119357300000001 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.720817+0000", 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4979e-05 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.720832+0000", 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018325100000000001 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:19.722664+0000", 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13323011100000001 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 123/118 [PushOp(5:8f611597:::benchmark_data_smithi125_106927_object13184:head, version: 112'770, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:8f611597:::benchmark_data_smithi125_106927_object13184:head@112'770, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:47.088236+0000", 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 212.38444306, 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101506837, 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088236+0000", 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088236+0000", 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088235+0000", 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3439999999999998e-05 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088268+0000", 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4888999999999998e-05 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088293+0000", 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2279999999999998e-06 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.088301+0000", 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.098876063 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.187177+0000", 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7178000000000001e-05 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.187195+0000", 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025481829999999999 2023-10-09T06:23:19.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:47.189743+0000", 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101506837 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:112623 5.9 e123/118 5:99669546:::benchmark_data_smithi125_106927_object8961:head v 123'7133, mlcod=0'0)", 2023-10-09T06:23:19.536 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:51.722285+0000", 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 207.750394046, 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.100577397, 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722285+0000", 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722285+0000", 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722284+0000", 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0580000000000004e-06 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722293+0000", 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4603999999999999e-05 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722307+0000", 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9009999999999997e-06 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.722314+0000", 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020506096000000001 2023-10-09T06:23:19.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.742820+0000", 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2904e-05 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.742843+0000", 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079980605999999996 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.822824+0000", 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8405999999999998e-05 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.822862+0000", 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.100577397 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:112625 5.9 e123/118 5:93c61ff1:::benchmark_data_smithi125_106927_object8963:head v 123'7134, mlcod=0'0)", 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:51.728655+0000", 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 207.74402356499999, 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10759102, 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728655+0000", 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728655+0000", 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728654+0000", 2023-10-09T06:23:19.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0324e-05 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728665+0000", 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.328e-05 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728678+0000", 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4970000000000004e-06 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.728685+0000", 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025068607999999999 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.753754+0000", 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5180999999999998e-05 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.753819+0000", 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.082338235999999995 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.836157+0000", 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8967000000000004e-05 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.836246+0000", 2023-10-09T06:23:19.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10759102 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 123/118 [PushOp(5:8a0ba734:::benchmark_data_smithi125_106927_object14307:head, version: 112'845, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:8a0ba734:::benchmark_data_smithi125_106927_object14307:head@112'845, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:51.744959+0000", 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 207.72772047500001, 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105680181, 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.547 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.744959+0000", 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.744959+0000", 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.744958+0000", 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7645999999999998e-05 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.744995+0000", 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7787000000000001e-05 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.745013+0000", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2760000000000006e-06 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.745022+0000", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.087120366000000005 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.832143+0000", 2023-10-09T06:23:19.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3237999999999999e-05 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.832156+0000", 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018482872000000001 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.850639+0000", 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105680181 2023-10-09T06:23:19.551 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 123/118 [PushOp(5:8617f1b4:::benchmark_data_smithi125_106927_object16096:head, version: 112'960, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:8617f1b4:::benchmark_data_smithi125_106927_object16096:head@112'960, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:58.395048+0000", 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 201.07763113300001, 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.121189439, 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395048+0000", 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395048+0000", 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395047+0000", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4891000000000002e-05 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395082+0000", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6827999999999999e-05 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395099+0000", 2023-10-09T06:23:19.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0113e-05 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.395109+0000", 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.11838383600000001 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.513493+0000", 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7000000000000002e-06 2023-10-09T06:23:19.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.513500+0000", 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027374999999999999 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:58.516237+0000", 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.121189439 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.556 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "MOSDPGPush(5.1 127/118 [PushOp(5:88b16516:::benchmark_data_smithi125_106927_object41906:head, version: 115'2609, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:88b16516:::benchmark_data_smithi125_106927_object41906:head@115'2609, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:43.456975+0000", 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 156.01570444199999, 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.115345009, 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.456975+0000", 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.456975+0000", 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.558 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.456973+0000", 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2790000000000002e-05 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.457016+0000", 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4902e-05 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.457041+0000", 2023-10-09T06:23:19.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2969000000000001e-05 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.457054+0000", 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.112874507 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.569928+0000", 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0260000000000007e-06 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.569936+0000", 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023830980000000002 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:43.572320+0000", 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.115345009 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.561 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:146098 5.d e127/118 5:b799a137:::benchmark_data_smithi125_106927_object42436:head v 127'9183, mlcod=0'0)", 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:44.382924+0000", 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 155.08975537699999, 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105806891, 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382924+0000", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382924+0000", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382922+0000", 2023-10-09T06:23:19.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7410000000000004e-06 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382932+0000", 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.526e-05 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382947+0000", 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2099999999999996e-06 2023-10-09T06:23:19.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.382955+0000", 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.016342605 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.399297+0000", 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8115000000000001e-05 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.399325+0000", 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.089303228999999998 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.488629+0000", 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010187300000000001 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:44.488730+0000", 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105806891 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:177809 5.6 e132/118 5:6402a09c:::benchmark_data_smithi125_106927_object74147:head v 132'10775, mlcod=0'0)", 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:21:36.741095+0000", 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 102.731584379, 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105900297, 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.567 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741095+0000", 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741095+0000", 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741094+0000", 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0956e-05 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741105+0000", 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5753e-05 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741120+0000", 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6930000000000001e-06 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.741127+0000", 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.034652545 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.775780+0000", 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6655000000000001e-05 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.775816+0000", 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.071132408999999994 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.846949+0000", 2023-10-09T06:23:19.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6235e-05 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:21:36.846995+0000", 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.105900297 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:200453 5.9 e136/118 5:9264957a:::benchmark_data_smithi125_106927_object96791:head v 136'12690, mlcod=0'0)", 2023-10-09T06:23:19.572 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:14.587346+0000", 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 64.885332761000001, 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10125485400000001, 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587346+0000", 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587346+0000", 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587345+0000", 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3974e-05 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587359+0000", 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7436999999999999e-05 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587376+0000", 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0229999999999992e-06 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.587384+0000", 2023-10-09T06:23:19.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021908210000000001 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.609292+0000", 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5851e-05 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.609318+0000", 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.079228326000000002 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.688547+0000", 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4432999999999999e-05 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.688601+0000", 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10125485400000001 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.577 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.44693.0:200457 5.9 e136/118 5:9135774d:::benchmark_data_smithi125_106927_object96795:head v 136'12691, mlcod=0'0)", 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:22:14.596619+0000", 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 64.876060408000001, 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101233462, 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:23:19.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596619+0000", 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596619+0000", 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596617+0000", 2023-10-09T06:23:19.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3861e-05 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596631+0000", 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4729000000000001e-05 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596646+0000", 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3930000000000002e-06 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.596653+0000", 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.020715765000000001 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.617369+0000", 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9524000000000001e-05 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:23:19.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.617399+0000", 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.080401331000000006 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.697800+0000", 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1986e-05 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:22:14.697852+0000", 2023-10-09T06:23:19.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.101233462 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:23:19.583 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:23:19.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:19.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:19.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:19.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:19.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:19.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:19.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:20.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:20.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:20.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:18.866136+0000 mon.a (mon.0) 754 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:20.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:18.866159+0000 mon.a (mon.0) 755 : cluster 1 Cluster is now healthy 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: audit 2023-10-09T06:23:18.877355+0000 mon.a (mon.0) 756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:18.877423+0000 mon.a (mon.0) 757 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:18.881016+0000 mgr.y (mgr.25723) 543 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:18.896188+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:19.323449+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:19.404950+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.128 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[80002]: cluster 2023-10-09T06:23:19.405120+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:18.866136+0000 mon.a (mon.0) 754 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:18.866159+0000 mon.a (mon.0) 755 : cluster 1 Cluster is now healthy 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: audit 2023-10-09T06:23:18.877355+0000 mon.a (mon.0) 756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:18.877423+0000 mon.a (mon.0) 757 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:18.881016+0000 mgr.y (mgr.25723) 543 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:18.896188+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:19.323449+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:19.404950+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:19 smithi125 bash[77299]: cluster 2023-10-09T06:23:19.405120+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:20.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:20.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:20.348 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:18.866136+0000 mon.a (mon.0) 754 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:20.348 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:18.866159+0000 mon.a (mon.0) 755 : cluster 1 Cluster is now healthy 2023-10-09T06:23:20.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: audit 2023-10-09T06:23:18.877355+0000 mon.a (mon.0) 756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:20.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:18.877423+0000 mon.a (mon.0) 757 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-09T06:23:20.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:18.881016+0000 mgr.y (mgr.25723) 543 : cluster 0 pgmap v513: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:20.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:18.896188+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:19.323449+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.350 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:19.404950+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.350 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:19 smithi155 bash[59495]: cluster 2023-10-09T06:23:19.405120+0000 osd.4 (osd.4) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-09T06:23:20.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:23:20.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:20.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:20.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:20.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:20.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:20.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:20.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:20.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:20.901 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T06:23:20.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:20.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:20.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:21.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:21.040 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:20 smithi125 bash[80002]: audit 2023-10-09T06:23:20.033076+0000 mon.a (mon.0) 758 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:23:21.040 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:20 smithi125 bash[77299]: audit 2023-10-09T06:23:20.033076+0000 mon.a (mon.0) 758 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:23:21.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:21.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:20 smithi155 bash[59495]: audit 2023-10-09T06:23:20.033076+0000 mon.a (mon.0) 758 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:23:21.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:21.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:21.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:21.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:21.332 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:21] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T06:23:21.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:21.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:21.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:21.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:21.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:21.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:21.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:21.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:23:21.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:21.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:21.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:21.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:21.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:21.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:22.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:22.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:22.239 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:21 smithi155 bash[59495]: cluster 2023-10-09T06:23:20.883715+0000 mgr.y (mgr.25723) 544 : cluster 0 pgmap v515: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:22.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:21 smithi155 bash[59495]: audit 2023-10-09T06:23:20.892256+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:23:22.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:21 smithi155 bash[59495]: cluster 2023-10-09T06:23:20.892352+0000 mon.a (mon.0) 760 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-09T06:23:22.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:21 smithi155 bash[59495]: cluster 2023-10-09T06:23:20.894386+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T06:23:22.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:22.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:22.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[77299]: cluster 2023-10-09T06:23:20.883715+0000 mgr.y (mgr.25723) 544 : cluster 0 pgmap v515: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:22.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[77299]: audit 2023-10-09T06:23:20.892256+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:23:22.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[77299]: cluster 2023-10-09T06:23:20.892352+0000 mon.a (mon.0) 760 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-09T06:23:22.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[77299]: cluster 2023-10-09T06:23:20.894386+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T06:23:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[80002]: cluster 2023-10-09T06:23:20.883715+0000 mgr.y (mgr.25723) 544 : cluster 0 pgmap v515: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[80002]: audit 2023-10-09T06:23:20.892256+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.125:0/2552738155' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:23:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[80002]: cluster 2023-10-09T06:23:20.892352+0000 mon.a (mon.0) 760 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-09T06:23:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:21 smithi125 bash[80002]: cluster 2023-10-09T06:23:20.894386+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e146 with expected crc 2023-10-09T06:23:22.394 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:22.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:22.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:22.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:22.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:22.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:23:22.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:22.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:22.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:23.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:23.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:23.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:23.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:23.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:23.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:23.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:23.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:23.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:23.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:23.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:23.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:23.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:23.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:23.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:23.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:23.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:23.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:23.978 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:23 smithi125 bash[77299]: cluster 2023-10-09T06:23:22.884176+0000 mgr.y (mgr.25723) 545 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:23.979 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:23 smithi125 bash[80002]: cluster 2023-10-09T06:23:22.884176+0000 mgr.y (mgr.25723) 545 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:23.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:24.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:24.116 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:23 smithi155 bash[59495]: cluster 2023-10-09T06:23:22.884176+0000 mgr.y (mgr.25723) 545 : cluster 0 pgmap v517: 89 pgs: 89 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:23:24.118 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T06:23:24.120 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:23:24.121 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:23:24.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:24.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:23:24.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:23:24.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_130224 2023-10-09T06:23:24.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:23:24.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:23:24.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:24.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:24.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:24.387 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:23:24.393 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:23:24.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:24.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:24.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:24.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:24.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:24.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:24.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:24.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:24.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:24.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:24.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:25.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:25.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:25.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 3072 3056 190.989 191 0.00282107 0.00521757 2023-10-09T06:23:25.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:25.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:25.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:25.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:25.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:25.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:25.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:25.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:25.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:25.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:25.791 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":517,"stamp":"2023-10-09T06:23:22.883889+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":1190420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2452236,"kb_used_data":744780,"kb_used_omap":30,"kb_used_meta":1707361,"kb_avail":747508980,"statfs":{"total":767960285184,"available":765449195520,"internally_reserved":0,"allocated":762654720,"data_stored":757985429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1748337775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.099538"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":126,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205265+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:23:04.205265+0000","last_peered":"2023-10-09T06:23:04.205265+0000","last_clean":"2023-10-09T06:23:04.205265+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:23:04.205265+0000","last_undegraded":"2023-10-09T06:23:04.205265+0000","last_fullsized":"2023-10-09T06:23:04.205265+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":116,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.199444+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:23:04.199444+0000","last_peered":"2023-10-09T06:23:04.199444+0000","last_clean":"2023-10-09T06:23:04.199444+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:23:04.199444+0000","last_undegraded":"2023-10-09T06:23:04.199444+0000","last_fullsized":"2023-10-09T06:23:04.199444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.371588+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:23:21.371588+0000","last_peered":"2023-10-09T06:23:21.371588+0000","last_clean":"2023-10-09T06:23:21.371588+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:23:21.371588+0000","last_undegraded":"2023-10-09T06:23:21.371588+0000","last_fullsized":"2023-10-09T06:23:21.371588+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.200558+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:23:04.200558+0000","last_peered":"2023-10-09T06:23:04.200558+0000","last_clean":"2023-10-09T06:23:04.200558+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:23:04.200558+0000","last_undegraded":"2023-10-09T06:23:04.200558+0000","last_fullsized":"2023-10-09T06:23:04.200558+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198785+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:23:04.198785+0000","last_peered":"2023-10-09T06:23:04.198785+0000","last_clean":"2023-10-09T06:23:04.198785+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:23:04.198785+0000","last_undegraded":"2023-10-09T06:23:04.198785+0000","last_fullsized":"2023-10-09T06:23:04.198785+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076546+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:23:04.076546+0000","last_peered":"2023-10-09T06:23:04.076546+0000","last_clean":"2023-10-09T06:23:04.076546+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:23:04.076546+0000","last_undegraded":"2023-10-09T06:23:04.076546+0000","last_fullsized":"2023-10-09T06:23:04.076546+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801246+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:23:02.801246+0000","last_peered":"2023-10-09T06:23:02.801246+0000","last_clean":"2023-10-09T06:23:02.801246+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:23:02.801246+0000","last_undegraded":"2023-10-09T06:23:02.801246+0000","last_fullsized":"2023-10-09T06:23:02.801246+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198678+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:23:04.198678+0000","last_peered":"2023-10-09T06:23:04.198678+0000","last_clean":"2023-10-09T06:23:04.198678+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:23:04.198678+0000","last_undegraded":"2023-10-09T06:23:04.198678+0000","last_fullsized":"2023-10-09T06:23:04.198678+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207603+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:23:04.207603+0000","last_peered":"2023-10-09T06:23:04.207603+0000","last_clean":"2023-10-09T06:23:04.207603+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:23:04.207603+0000","last_undegraded":"2023-10-09T06:23:04.207603+0000","last_fullsized":"2023-10-09T06:23:04.207603+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198635+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:23:04.198635+0000","last_peered":"2023-10-09T06:23:04.198635+0000","last_clean":"2023-10-09T06:23:04.198635+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:23:04.198635+0000","last_undegraded":"2023-10-09T06:23:04.198635+0000","last_fullsized":"2023-10-09T06:23:04.198635+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":43,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210647+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:23:04.210647+0000","last_peered":"2023-10-09T06:23:04.210647+0000","last_clean":"2023-10-09T06:23:04.210647+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:23:04.210647+0000","last_undegraded":"2023-10-09T06:23:04.210647+0000","last_fullsized":"2023-10-09T06:23:04.210647+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400999+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:23:19.400999+0000","last_peered":"2023-10-09T06:23:19.400999+0000","last_clean":"2023-10-09T06:23:19.400999+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:23:19.400999+0000","last_undegraded":"2023-10-09T06:23:19.400999+0000","last_fullsized":"2023-10-09T06:23:19.400999+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":155,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206141+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:23:04.206141+0000","last_peered":"2023-10-09T06:23:04.206141+0000","last_clean":"2023-10-09T06:23:04.206141+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:23:04.206141+0000","last_undegraded":"2023-10-09T06:23:04.206141+0000","last_fullsized":"2023-10-09T06:23:04.206141+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206363+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:23:04.206363+0000","last_peered":"2023-10-09T06:23:04.206363+0000","last_clean":"2023-10-09T06:23:04.206363+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:23:04.206363+0000","last_undegraded":"2023-10-09T06:23:04.206363+0000","last_fullsized":"2023-10-09T06:23:04.206363+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":43,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210739+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:23:04.210739+0000","last_peered":"2023-10-09T06:23:04.210739+0000","last_clean":"2023-10-09T06:23:04.210739+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:23:04.210739+0000","last_undegraded":"2023-10-09T06:23:04.210739+0000","last_fullsized":"2023-10-09T06:23:04.210739+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.199360+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:23:04.199360+0000","last_peered":"2023-10-09T06:23:04.199360+0000","last_clean":"2023-10-09T06:23:04.199360+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:23:04.199360+0000","last_undegraded":"2023-10-09T06:23:04.199360+0000","last_fullsized":"2023-10-09T06:23:04.199360+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801308+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:23:02.801308+0000","last_peered":"2023-10-09T06:23:02.801308+0000","last_clean":"2023-10-09T06:23:02.801308+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:23:02.801308+0000","last_undegraded":"2023-10-09T06:23:02.801308+0000","last_fullsized":"2023-10-09T06:23:02.801308+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":127,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.208071+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:23:04.208071+0000","last_peered":"2023-10-09T06:23:04.208071+0000","last_clean":"2023-10-09T06:23:04.208071+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:23:04.208071+0000","last_undegraded":"2023-10-09T06:23:04.208071+0000","last_fullsized":"2023-10-09T06:23:04.208071+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":37,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210492+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:23:04.210492+0000","last_peered":"2023-10-09T06:23:04.210492+0000","last_clean":"2023-10-09T06:23:04.210492+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:23:04.210492+0000","last_undegraded":"2023-10-09T06:23:04.210492+0000","last_fullsized":"2023-10-09T06:23:04.210492+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801195+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:23:02.801195+0000","last_peered":"2023-10-09T06:23:02.801195+0000","last_clean":"2023-10-09T06:23:02.801195+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:23:02.801195+0000","last_undegraded":"2023-10-09T06:23:02.801195+0000","last_fullsized":"2023-10-09T06:23:02.801195+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":119,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076164+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:23:04.076164+0000","last_peered":"2023-10-09T06:23:04.076164+0000","last_clean":"2023-10-09T06:23:04.076164+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:23:04.076164+0000","last_undegraded":"2023-10-09T06:23:04.076164+0000","last_fullsized":"2023-10-09T06:23:04.076164+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.199697+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:23:04.199697+0000","last_peered":"2023-10-09T06:23:04.199697+0000","last_clean":"2023-10-09T06:23:04.199697+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:23:04.199697+0000","last_undegraded":"2023-10-09T06:23:04.199697+0000","last_fullsized":"2023-10-09T06:23:04.199697+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207073+0000","last_change":"2023-10-09T06:23:02.818534+0000","last_active":"2023-10-09T06:23:04.207073+0000","last_peered":"2023-10-09T06:23:04.207073+0000","last_clean":"2023-10-09T06:23:04.207073+0000","last_became_active":"2023-10-09T06:23:02.816918+0000","last_became_peered":"2023-10-09T06:23:02.816918+0000","last_unstale":"2023-10-09T06:23:04.207073+0000","last_undegraded":"2023-10-09T06:23:04.207073+0000","last_fullsized":"2023-10-09T06:23:04.207073+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":41,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.211166+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:23:04.211166+0000","last_peered":"2023-10-09T06:23:04.211166+0000","last_clean":"2023-10-09T06:23:04.211166+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:23:04.211166+0000","last_undegraded":"2023-10-09T06:23:04.211166+0000","last_fullsized":"2023-10-09T06:23:04.211166+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207079+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:23:04.207079+0000","last_peered":"2023-10-09T06:23:04.207079+0000","last_clean":"2023-10-09T06:23:04.207079+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:23:04.207079+0000","last_undegraded":"2023-10-09T06:23:04.207079+0000","last_fullsized":"2023-10-09T06:23:04.207079+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":12,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.807545+0000","last_change":"2023-10-09T06:23:02.810028+0000","last_active":"2023-10-09T06:23:05.807545+0000","last_peered":"2023-10-09T06:23:05.807545+0000","last_clean":"2023-10-09T06:23:05.807545+0000","last_became_active":"2023-10-09T06:23:02.809179+0000","last_became_peered":"2023-10-09T06:23:02.809179+0000","last_unstale":"2023-10-09T06:23:05.807545+0000","last_undegraded":"2023-10-09T06:23:05.807545+0000","last_fullsized":"2023-10-09T06:23:05.807545+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:49.851184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":41,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.211194+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:23:04.211194+0000","last_peered":"2023-10-09T06:23:04.211194+0000","last_clean":"2023-10-09T06:23:04.211194+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:23:04.211194+0000","last_undegraded":"2023-10-09T06:23:04.211194+0000","last_fullsized":"2023-10-09T06:23:04.211194+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205544+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:23:04.205544+0000","last_peered":"2023-10-09T06:23:04.205544+0000","last_clean":"2023-10-09T06:23:04.205544+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:23:04.205544+0000","last_undegraded":"2023-10-09T06:23:04.205544+0000","last_fullsized":"2023-10-09T06:23:04.205544+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.209903+0000","last_change":"2023-10-09T06:23:02.818119+0000","last_active":"2023-10-09T06:23:04.209903+0000","last_peered":"2023-10-09T06:23:04.209903+0000","last_clean":"2023-10-09T06:23:04.209903+0000","last_became_active":"2023-10-09T06:23:02.817802+0000","last_became_peered":"2023-10-09T06:23:02.817802+0000","last_unstale":"2023-10-09T06:23:04.209903+0000","last_undegraded":"2023-10-09T06:23:04.209903+0000","last_fullsized":"2023-10-09T06:23:04.209903+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":41,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210966+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:23:04.210966+0000","last_peered":"2023-10-09T06:23:04.210966+0000","last_clean":"2023-10-09T06:23:04.210966+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:23:04.210966+0000","last_undegraded":"2023-10-09T06:23:04.210966+0000","last_fullsized":"2023-10-09T06:23:04.210966+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076552+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:23:04.076552+0000","last_peered":"2023-10-09T06:23:04.076552+0000","last_clean":"2023-10-09T06:23:04.076552+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:23:04.076552+0000","last_undegraded":"2023-10-09T06:23:04.076552+0000","last_fullsized":"2023-10-09T06:23:04.076552+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":11,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:03.188605+0000","last_change":"2023-10-09T06:23:02.807021+0000","last_active":"2023-10-09T06:23:03.188605+0000","last_peered":"2023-10-09T06:23:03.188605+0000","last_clean":"2023-10-09T06:23:03.188605+0000","last_became_active":"2023-10-09T06:23:02.806707+0000","last_became_peered":"2023-10-09T06:23:02.806707+0000","last_unstale":"2023-10-09T06:23:03.188605+0000","last_undegraded":"2023-10-09T06:23:03.188605+0000","last_fullsized":"2023-10-09T06:23:03.188605+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:19.835325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207619+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:23:04.207619+0000","last_peered":"2023-10-09T06:23:04.207619+0000","last_clean":"2023-10-09T06:23:04.207619+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:23:04.207619+0000","last_undegraded":"2023-10-09T06:23:04.207619+0000","last_fullsized":"2023-10-09T06:23:04.207619+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801043+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:23:02.801043+0000","last_peered":"2023-10-09T06:23:02.801043+0000","last_clean":"2023-10-09T06:23:02.801043+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:23:02.801043+0000","last_undegraded":"2023-10-09T06:23:02.801043+0000","last_fullsized":"2023-10-09T06:23:02.801043+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206582+0000","last_change":"2023-10-09T06:23:02.820523+0000","last_active":"2023-10-09T06:23:04.206582+0000","last_peered":"2023-10-09T06:23:04.206582+0000","last_clean":"2023-10-09T06:23:04.206582+0000","last_became_active":"2023-10-09T06:23:02.820107+0000","last_became_peered":"2023-10-09T06:23:02.820107+0000","last_unstale":"2023-10-09T06:23:04.206582+0000","last_undegraded":"2023-10-09T06:23:04.206582+0000","last_fullsized":"2023-10-09T06:23:04.206582+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":115,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.199237+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:23:04.199237+0000","last_peered":"2023-10-09T06:23:04.199237+0000","last_clean":"2023-10-09T06:23:04.199237+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:23:04.199237+0000","last_undegraded":"2023-10-09T06:23:04.199237+0000","last_fullsized":"2023-10-09T06:23:04.199237+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205696+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:23:04.205696+0000","last_peered":"2023-10-09T06:23:04.205696+0000","last_clean":"2023-10-09T06:23:04.205696+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:23:04.205696+0000","last_undegraded":"2023-10-09T06:23:04.205696+0000","last_fullsized":"2023-10-09T06:23:04.205696+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":13,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.876234+0000","last_change":"2023-10-09T06:23:02.812031+0000","last_active":"2023-10-09T06:23:18.876234+0000","last_peered":"2023-10-09T06:23:18.876234+0000","last_clean":"2023-10-09T06:23:18.876234+0000","last_became_active":"2023-10-09T06:23:02.811742+0000","last_became_peered":"2023-10-09T06:23:02.811742+0000","last_unstale":"2023-10-09T06:23:18.876234+0000","last_undegraded":"2023-10-09T06:23:18.876234+0000","last_fullsized":"2023-10-09T06:23:18.876234+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.791030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076348+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:23:04.076348+0000","last_peered":"2023-10-09T06:23:04.076348+0000","last_clean":"2023-10-09T06:23:04.076348+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:23:04.076348+0000","last_undegraded":"2023-10-09T06:23:04.076348+0000","last_fullsized":"2023-10-09T06:23:04.076348+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205959+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:23:04.205959+0000","last_peered":"2023-10-09T06:23:04.205959+0000","last_clean":"2023-10-09T06:23:04.205959+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:23:04.205959+0000","last_undegraded":"2023-10-09T06:23:04.205959+0000","last_fullsized":"2023-10-09T06:23:04.205959+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":12,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.807695+0000","last_change":"2023-10-09T06:23:02.813423+0000","last_active":"2023-10-09T06:23:05.807695+0000","last_peered":"2023-10-09T06:23:05.807695+0000","last_clean":"2023-10-09T06:23:05.807695+0000","last_became_active":"2023-10-09T06:23:02.813075+0000","last_became_peered":"2023-10-09T06:23:02.813075+0000","last_unstale":"2023-10-09T06:23:05.807695+0000","last_undegraded":"2023-10-09T06:23:05.807695+0000","last_fullsized":"2023-10-09T06:23:05.807695+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:07.790690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205158+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:23:04.205158+0000","last_peered":"2023-10-09T06:23:04.205158+0000","last_clean":"2023-10-09T06:23:04.205158+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:23:04.205158+0000","last_undegraded":"2023-10-09T06:23:04.205158+0000","last_fullsized":"2023-10-09T06:23:04.205158+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206842+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:23:04.206842+0000","last_peered":"2023-10-09T06:23:04.206842+0000","last_clean":"2023-10-09T06:23:04.206842+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:23:04.206842+0000","last_undegraded":"2023-10-09T06:23:04.206842+0000","last_fullsized":"2023-10-09T06:23:04.206842+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207948+0000","last_change":"2023-10-09T06:23:02.819776+0000","last_active":"2023-10-09T06:23:04.207948+0000","last_peered":"2023-10-09T06:23:04.207948+0000","last_clean":"2023-10-09T06:23:04.207948+0000","last_became_active":"2023-10-09T06:23:02.819408+0000","last_became_peered":"2023-10-09T06:23:02.819408+0000","last_unstale":"2023-10-09T06:23:04.207948+0000","last_undegraded":"2023-10-09T06:23:04.207948+0000","last_fullsized":"2023-10-09T06:23:04.207948+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":136,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878706+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:23:18.878706+0000","last_peered":"2023-10-09T06:23:18.878706+0000","last_clean":"2023-10-09T06:23:18.878706+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:23:18.878706+0000","last_undegraded":"2023-10-09T06:23:18.878706+0000","last_fullsized":"2023-10-09T06:23:18.878706+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":94,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401347+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:23:19.401347+0000","last_peered":"2023-10-09T06:23:19.401347+0000","last_clean":"2023-10-09T06:23:19.401347+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:23:19.401347+0000","last_undegraded":"2023-10-09T06:23:19.401347+0000","last_fullsized":"2023-10-09T06:23:19.401347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":11,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:03.188549+0000","last_change":"2023-10-09T06:23:02.808018+0000","last_active":"2023-10-09T06:23:03.188549+0000","last_peered":"2023-10-09T06:23:03.188549+0000","last_clean":"2023-10-09T06:23:03.188549+0000","last_became_active":"2023-10-09T06:23:02.807707+0000","last_became_peered":"2023-10-09T06:23:02.807707+0000","last_unstale":"2023-10-09T06:23:03.188549+0000","last_undegraded":"2023-10-09T06:23:03.188549+0000","last_fullsized":"2023-10-09T06:23:03.188549+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.598902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":994,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400899+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:23:19.400899+0000","last_peered":"2023-10-09T06:23:19.400899+0000","last_clean":"2023-10-09T06:23:19.400899+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:23:19.400899+0000","last_undegraded":"2023-10-09T06:23:19.400899+0000","last_fullsized":"2023-10-09T06:23:19.400899+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.208183+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:23:04.208183+0000","last_peered":"2023-10-09T06:23:04.208183+0000","last_clean":"2023-10-09T06:23:04.208183+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:23:04.208183+0000","last_undegraded":"2023-10-09T06:23:04.208183+0000","last_fullsized":"2023-10-09T06:23:04.208183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400887+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:23:19.400887+0000","last_peered":"2023-10-09T06:23:19.400887+0000","last_clean":"2023-10-09T06:23:19.400887+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:23:19.400887+0000","last_undegraded":"2023-10-09T06:23:19.400887+0000","last_fullsized":"2023-10-09T06:23:19.400887+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":12,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.807799+0000","last_change":"2023-10-09T06:23:02.809922+0000","last_active":"2023-10-09T06:23:05.807799+0000","last_peered":"2023-10-09T06:23:05.807799+0000","last_clean":"2023-10-09T06:23:05.807799+0000","last_became_active":"2023-10-09T06:23:02.809657+0000","last_became_peered":"2023-10-09T06:23:02.809657+0000","last_unstale":"2023-10-09T06:23:05.807799+0000","last_undegraded":"2023-10-09T06:23:05.807799+0000","last_fullsized":"2023-10-09T06:23:05.807799+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:22.193166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":726,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207291+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:23:04.207291+0000","last_peered":"2023-10-09T06:23:04.207291+0000","last_clean":"2023-10-09T06:23:04.207291+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:23:04.207291+0000","last_undegraded":"2023-10-09T06:23:04.207291+0000","last_fullsized":"2023-10-09T06:23:04.207291+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206137+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:23:04.206137+0000","last_peered":"2023-10-09T06:23:04.206137+0000","last_clean":"2023-10-09T06:23:04.206137+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:23:04.206137+0000","last_undegraded":"2023-10-09T06:23:04.206137+0000","last_fullsized":"2023-10-09T06:23:04.206137+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400750+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:23:19.400750+0000","last_peered":"2023-10-09T06:23:19.400750+0000","last_clean":"2023-10-09T06:23:19.400750+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:23:19.400750+0000","last_undegraded":"2023-10-09T06:23:19.400750+0000","last_fullsized":"2023-10-09T06:23:19.400750+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210557+0000","last_change":"2023-10-09T06:23:02.818151+0000","last_active":"2023-10-09T06:23:04.210557+0000","last_peered":"2023-10-09T06:23:04.210557+0000","last_clean":"2023-10-09T06:23:04.210557+0000","last_became_active":"2023-10-09T06:23:02.816332+0000","last_became_peered":"2023-10-09T06:23:02.816332+0000","last_unstale":"2023-10-09T06:23:04.210557+0000","last_undegraded":"2023-10-09T06:23:04.210557+0000","last_fullsized":"2023-10-09T06:23:04.210557+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":505,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.876514+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:23:18.876514+0000","last_peered":"2023-10-09T06:23:18.876514+0000","last_clean":"2023-10-09T06:23:18.876514+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:23:18.876514+0000","last_undegraded":"2023-10-09T06:23:18.876514+0000","last_fullsized":"2023-10-09T06:23:18.876514+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":93,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400413+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:23:19.400413+0000","last_peered":"2023-10-09T06:23:19.400413+0000","last_clean":"2023-10-09T06:23:19.400413+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:23:19.400413+0000","last_undegraded":"2023-10-09T06:23:19.400413+0000","last_fullsized":"2023-10-09T06:23:19.400413+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401371+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:23:19.401371+0000","last_peered":"2023-10-09T06:23:19.401371+0000","last_clean":"2023-10-09T06:23:19.401371+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:23:19.401371+0000","last_undegraded":"2023-10-09T06:23:19.401371+0000","last_fullsized":"2023-10-09T06:23:19.401371+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207690+0000","last_change":"2023-10-09T06:23:02.821017+0000","last_active":"2023-10-09T06:23:04.207690+0000","last_peered":"2023-10-09T06:23:04.207690+0000","last_clean":"2023-10-09T06:23:04.207690+0000","last_became_active":"2023-10-09T06:23:02.820708+0000","last_became_peered":"2023-10-09T06:23:02.820708+0000","last_unstale":"2023-10-09T06:23:04.207690+0000","last_undegraded":"2023-10-09T06:23:04.207690+0000","last_fullsized":"2023-10-09T06:23:04.207690+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":493,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207412+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:23:04.207412+0000","last_peered":"2023-10-09T06:23:04.207412+0000","last_clean":"2023-10-09T06:23:04.207412+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:23:04.207412+0000","last_undegraded":"2023-10-09T06:23:04.207412+0000","last_fullsized":"2023-10-09T06:23:04.207412+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":341,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210325+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:23:04.210325+0000","last_peered":"2023-10-09T06:23:04.210325+0000","last_clean":"2023-10-09T06:23:04.210325+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:23:04.210325+0000","last_undegraded":"2023-10-09T06:23:04.210325+0000","last_fullsized":"2023-10-09T06:23:04.210325+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":108,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198753+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:23:04.198753+0000","last_peered":"2023-10-09T06:23:04.198753+0000","last_clean":"2023-10-09T06:23:04.198753+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:23:04.198753+0000","last_undegraded":"2023-10-09T06:23:04.198753+0000","last_fullsized":"2023-10-09T06:23:04.198753+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076452+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:23:04.076452+0000","last_peered":"2023-10-09T06:23:04.076452+0000","last_clean":"2023-10-09T06:23:04.076452+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:23:04.076452+0000","last_undegraded":"2023-10-09T06:23:04.076452+0000","last_fullsized":"2023-10-09T06:23:04.076452+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":13,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400686+0000","last_change":"2023-10-09T06:23:02.811831+0000","last_active":"2023-10-09T06:23:19.400686+0000","last_peered":"2023-10-09T06:23:19.400686+0000","last_clean":"2023-10-09T06:23:19.400686+0000","last_became_active":"2023-10-09T06:23:02.811445+0000","last_became_peered":"2023-10-09T06:23:02.811445+0000","last_unstale":"2023-10-09T06:23:19.400686+0000","last_undegraded":"2023-10-09T06:23:19.400686+0000","last_fullsized":"2023-10-09T06:23:19.400686+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:28.021647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":66,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210070+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:23:04.210070+0000","last_peered":"2023-10-09T06:23:04.210070+0000","last_clean":"2023-10-09T06:23:04.210070+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:23:04.210070+0000","last_undegraded":"2023-10-09T06:23:04.210070+0000","last_fullsized":"2023-10-09T06:23:04.210070+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.802355+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:23:02.802355+0000","last_peered":"2023-10-09T06:23:02.802355+0000","last_clean":"2023-10-09T06:23:02.802355+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:23:02.802355+0000","last_undegraded":"2023-10-09T06:23:02.802355+0000","last_fullsized":"2023-10-09T06:23:02.802355+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801492+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:23:02.801492+0000","last_peered":"2023-10-09T06:23:02.801492+0000","last_clean":"2023-10-09T06:23:02.801492+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:23:02.801492+0000","last_undegraded":"2023-10-09T06:23:02.801492+0000","last_fullsized":"2023-10-09T06:23:02.801492+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":13,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400669+0000","last_change":"2023-10-09T06:23:02.810207+0000","last_active":"2023-10-09T06:23:19.400669+0000","last_peered":"2023-10-09T06:23:19.400669+0000","last_clean":"2023-10-09T06:23:19.400669+0000","last_became_active":"2023-10-09T06:23:02.809844+0000","last_became_peered":"2023-10-09T06:23:02.809844+0000","last_unstale":"2023-10-09T06:23:19.400669+0000","last_undegraded":"2023-10-09T06:23:19.400669+0000","last_fullsized":"2023-10-09T06:23:19.400669+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:38.667224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":55,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210913+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:23:04.210913+0000","last_peered":"2023-10-09T06:23:04.210913+0000","last_clean":"2023-10-09T06:23:04.210913+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:23:04.210913+0000","last_undegraded":"2023-10-09T06:23:04.210913+0000","last_fullsized":"2023-10-09T06:23:04.210913+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":93,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400793+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:23:19.400793+0000","last_peered":"2023-10-09T06:23:19.400793+0000","last_clean":"2023-10-09T06:23:19.400793+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:23:19.400793+0000","last_undegraded":"2023-10-09T06:23:19.400793+0000","last_fullsized":"2023-10-09T06:23:19.400793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":107,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801602+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:23:02.801602+0000","last_peered":"2023-10-09T06:23:02.801602+0000","last_clean":"2023-10-09T06:23:02.801602+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:23:02.801602+0000","last_undegraded":"2023-10-09T06:23:02.801602+0000","last_fullsized":"2023-10-09T06:23:02.801602+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205264+0000","last_change":"2023-10-09T06:23:02.821182+0000","last_active":"2023-10-09T06:23:04.205264+0000","last_peered":"2023-10-09T06:23:04.205264+0000","last_clean":"2023-10-09T06:23:04.205264+0000","last_became_active":"2023-10-09T06:23:02.820839+0000","last_became_peered":"2023-10-09T06:23:02.820839+0000","last_unstale":"2023-10-09T06:23:04.205264+0000","last_undegraded":"2023-10-09T06:23:04.205264+0000","last_fullsized":"2023-10-09T06:23:04.205264+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":142,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.880142+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:23:18.880142+0000","last_peered":"2023-10-09T06:23:18.880142+0000","last_clean":"2023-10-09T06:23:18.880142+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:23:18.880142+0000","last_undegraded":"2023-10-09T06:23:18.880142+0000","last_fullsized":"2023-10-09T06:23:18.880142+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":450,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401243+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:23:19.401243+0000","last_peered":"2023-10-09T06:23:19.401243+0000","last_clean":"2023-10-09T06:23:19.401243+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:23:19.401243+0000","last_undegraded":"2023-10-09T06:23:19.401243+0000","last_fullsized":"2023-10-09T06:23:19.401243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198851+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:23:04.198851+0000","last_peered":"2023-10-09T06:23:04.198851+0000","last_clean":"2023-10-09T06:23:04.198851+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:23:04.198851+0000","last_undegraded":"2023-10-09T06:23:04.198851+0000","last_fullsized":"2023-10-09T06:23:04.198851+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":11,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.210420+0000","last_change":"2023-10-09T06:23:02.819624+0000","last_active":"2023-10-09T06:23:04.210420+0000","last_peered":"2023-10-09T06:23:04.210420+0000","last_clean":"2023-10-09T06:23:04.210420+0000","last_became_active":"2023-10-09T06:23:02.818630+0000","last_became_peered":"2023-10-09T06:23:02.818630+0000","last_unstale":"2023-10-09T06:23:04.210420+0000","last_undegraded":"2023-10-09T06:23:04.210420+0000","last_fullsized":"2023-10-09T06:23:04.210420+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":841,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.076447+0000","last_change":"2023-10-09T06:22:46.890972+0000","last_active":"2023-10-09T06:23:04.076447+0000","last_peered":"2023-10-09T06:23:04.076447+0000","last_clean":"2023-10-09T06:23:04.076447+0000","last_became_active":"2023-10-09T06:22:46.889916+0000","last_became_peered":"2023-10-09T06:22:46.889916+0000","last_unstale":"2023-10-09T06:23:04.076447+0000","last_undegraded":"2023-10-09T06:23:04.076447+0000","last_fullsized":"2023-10-09T06:23:04.076447+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":99,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801417+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:23:02.801417+0000","last_peered":"2023-10-09T06:23:02.801417+0000","last_clean":"2023-10-09T06:23:02.801417+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:23:02.801417+0000","last_undegraded":"2023-10-09T06:23:02.801417+0000","last_fullsized":"2023-10-09T06:23:02.801417+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":94,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205204+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:23:04.205204+0000","last_peered":"2023-10-09T06:23:04.205204+0000","last_clean":"2023-10-09T06:23:04.205204+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:23:04.205204+0000","last_undegraded":"2023-10-09T06:23:04.205204+0000","last_fullsized":"2023-10-09T06:23:04.205204+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":102,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.200592+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:23:04.200592+0000","last_peered":"2023-10-09T06:23:04.200592+0000","last_clean":"2023-10-09T06:23:04.200592+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:23:04.200592+0000","last_undegraded":"2023-10-09T06:23:04.200592+0000","last_fullsized":"2023-10-09T06:23:04.200592+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.876341+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:23:18.876341+0000","last_peered":"2023-10-09T06:23:18.876341+0000","last_clean":"2023-10-09T06:23:18.876341+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:23:18.876341+0000","last_undegraded":"2023-10-09T06:23:18.876341+0000","last_fullsized":"2023-10-09T06:23:18.876341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":112,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.207898+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:23:04.207898+0000","last_peered":"2023-10-09T06:23:04.207898+0000","last_clean":"2023-10-09T06:23:04.207898+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:23:04.207898+0000","last_undegraded":"2023-10-09T06:23:04.207898+0000","last_fullsized":"2023-10-09T06:23:04.207898+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.206537+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:23:04.206537+0000","last_peered":"2023-10-09T06:23:04.206537+0000","last_clean":"2023-10-09T06:23:04.206537+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:23:04.206537+0000","last_undegraded":"2023-10-09T06:23:04.206537+0000","last_fullsized":"2023-10-09T06:23:04.206537+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":97,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.208242+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:23:04.208242+0000","last_peered":"2023-10-09T06:23:04.208242+0000","last_clean":"2023-10-09T06:23:04.208242+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:23:04.208242+0000","last_undegraded":"2023-10-09T06:23:04.208242+0000","last_fullsized":"2023-10-09T06:23:04.208242+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.205378+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:23:04.205378+0000","last_peered":"2023-10-09T06:23:04.205378+0000","last_clean":"2023-10-09T06:23:04.205378+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:23:04.205378+0000","last_undegraded":"2023-10-09T06:23:04.205378+0000","last_fullsized":"2023-10-09T06:23:04.205378+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":123,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.199890+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:23:04.199890+0000","last_peered":"2023-10-09T06:23:04.199890+0000","last_clean":"2023-10-09T06:23:04.199890+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:23:04.199890+0000","last_undegraded":"2023-10-09T06:23:04.199890+0000","last_fullsized":"2023-10-09T06:23:04.199890+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.198666+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:23:04.198666+0000","last_peered":"2023-10-09T06:23:04.198666+0000","last_clean":"2023-10-09T06:23:04.198666+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:23:04.198666+0000","last_undegraded":"2023-10-09T06:23:04.198666+0000","last_fullsized":"2023-10-09T06:23:04.198666+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":99,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.801190+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:23:02.801190+0000","last_peered":"2023-10-09T06:23:02.801190+0000","last_clean":"2023-10-09T06:23:02.801190+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:23:02.801190+0000","last_undegraded":"2023-10-09T06:23:02.801190+0000","last_fullsized":"2023-10-09T06:23:02.801190+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-09T06:23:04.204981+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:23:04.204981+0000","last_peered":"2023-10-09T06:23:04.204981+0000","last_clean":"2023-10-09T06:23:04.204981+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:23:04.204981+0000","last_undegraded":"2023-10-09T06:23:04.204981+0000","last_fullsized":"2023-10-09T06:23:04.204981+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430504,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299196,"kb_used_data":1972,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93445956,"statfs":{"total":95995035648,"available":95688658944,"internally_reserved":0,"allocated":2019328,"data_stored":1493835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.4769999999999999,"15min":1.4039999999999999},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7969999999999997,"5min":107.36799999999999,"15min":107.36799999999999},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.4359999999999999,"15min":1.391},"min":{"1min":0.371,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.8480000000000008,"5min":105.89,"15min":105.89},"last":0.92200000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:23:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":2.883,"15min":1.55},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.7199999999999998,"5min":104.825,"15min":104.825},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.296,"5min":2.46,"15min":1.4430000000000001},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.6799999999999997,"5min":104.637,"15min":104.637},"last":1.3640000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:22:22 2023","interfaces":[{"interface":"back","average":{"1min":5.8419999999999996,"5min":4.1180000000000003,"15min":1.992},"min":{"1min":0.46700000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":61.822000000000003,"5min":104.992,"15min":104.992},"last":0.996},{"interface":"front","average":{"1min":5.3449999999999998,"5min":4.0529999999999999,"15min":2.012},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":65.064999999999998,"5min":104.691,"15min":104.691},"last":1.4390000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":1.4490000000000001,"15min":1.1559999999999999},"min":{"1min":0.48899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.46,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.163,"15min":1.151},"min":{"1min":0.40999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":4.3609999999999998,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":1.111}]},{"osd":4,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.804,"15min":1.214},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.029999999999999,"5min":11.029999999999999,"15min":11.029999999999999},"last":0.79800000000000004},{"interface":"front","average":{"1min":2.754,"5min":1.9370000000000001,"15min":1.2769999999999999},"min":{"1min":0.36699999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":1.296}]},{"osd":5,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.8680000000000001,"15min":1.3009999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.3339999999999996,"5min":11.996,"15min":11.996},"last":1.175},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.837,"15min":1.3080000000000001},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.9019999999999992,"5min":13.634,"15min":13.634},"last":1.012}]},{"osd":6,"last update":"Mon Oct 9 06:22:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.4359999999999999,"15min":1.1990000000000001},"min":{"1min":0.40000000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":8.9260000000000002,"5min":9.9510000000000005,"15min":9.9510000000000005},"last":1.1279999999999999},{"interface":"front","average":{"1min":2.02,"5min":1.708,"15min":1.282},"min":{"1min":0.53500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.5899999999999999,"5min":8.6199999999999992,"15min":8.6199999999999992},"last":0.72499999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626736,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303268,"kb_used_data":1948,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441884,"statfs":{"total":95995035648,"available":95684489216,"internally_reserved":0,"allocated":1994752,"data_stored":1485341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:33 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.706,"15min":1.1639999999999999},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.1959999999999997,"5min":32.289999999999999,"15min":32.289999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.4219999999999999,"5min":1.5549999999999999,"15min":1.1359999999999999},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.2199999999999998,"5min":31.454999999999998,"15min":31.454999999999998},"last":0.81499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.6219999999999999,"15min":1.454},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9249999999999998,"5min":137.93899999999999,"15min":137.93899999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.1379999999999999,"5min":2.673,"15min":1.4930000000000001},"min":{"1min":0.35199999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.1529999999999996,"5min":138.02500000000001,"15min":138.02500000000001},"last":0.58499999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:22:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":2.9649999999999999,"15min":1.637},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.952,"5min":95.540000000000006,"15min":95.540000000000006},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.1440000000000001,"5min":3.073,"15min":1.655},"min":{"1min":0.318,"5min":0.309,"15min":0.309},"max":{"1min":6.8250000000000002,"5min":96.352000000000004,"15min":96.352000000000004},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.3109999999999999,"15min":1.137},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.2290000000000001,"5min":7.3410000000000002,"15min":7.3410000000000002},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.542,"5min":1.1259999999999999,"15min":1.046},"min":{"1min":0.38,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.4860000000000007,"5min":9.4860000000000007,"15min":9.4860000000000007},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.6060000000000001,"15min":1.163},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.3310000000000004,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.403,"5min":1.746,"15min":1.2470000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.851,"5min":10.300000000000001,"15min":10.300000000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.8220000000000001,"15min":1.2889999999999999},"min":{"1min":0.46000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.766,"5min":11.754,"15min":11.754},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.8049999999999999,"15min":1.3029999999999999},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":9.0419999999999998,"5min":12.295999999999999,"15min":12.295999999999999},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 06:22:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.7709999999999999,"15min":1.3200000000000001},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.4179999999999993,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.88900000000000001},{"interface":"front","average":{"1min":2.5070000000000001,"5min":1.921,"15min":1.3959999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.102,"5min":11.102,"15min":11.102},"last":0.91600000000000004}]}]},{"osd":1,"up_from":93,"seq":399431958646,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131664,"kb_used_data":1544,"kb_used_omap":7,"kb_used_meta":130104,"kb_avail":93613488,"statfs":{"total":95995035648,"available":95860211712,"internally_reserved":0,"allocated":1581056,"data_stored":923744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":133226896},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.4350000000000001,"15min":0.94999999999999996},"min":{"1min":0.45000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.698,"5min":31.969000000000001,"15min":31.969000000000001},"last":0.629},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.8839999999999999,"15min":1.085},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.8879999999999999,"5min":40.667999999999999,"15min":40.667999999999999},"last":0.81999999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.319,"15min":0.89200000000000002},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":4.0259999999999998,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.3580000000000001,"15min":1.004},"min":{"1min":0.39600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6959999999999997,"5min":8.3960000000000008,"15min":8.3960000000000008},"last":0.79600000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.2909999999999999,"15min":0.98899999999999999},"min":{"1min":0.34999999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.6539999999999999,"5min":16.390000000000001,"15min":16.390000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.1220000000000001,"15min":0.88400000000000001},"min":{"1min":0.34100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.657,"5min":16.555,"15min":16.555},"last":0.93799999999999994}]},{"osd":4,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.5069999999999999,"15min":1.1619999999999999},"min":{"1min":0.48699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.7469999999999999,"5min":6.7469999999999999,"15min":6.7469999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.6519999999999999,"15min":1.2110000000000001},"min":{"1min":0.441,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6340000000000003,"5min":7.3639999999999999,"15min":7.3639999999999999},"last":1.095}]},{"osd":5,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.748,"15min":1.2609999999999999},"min":{"1min":0.49299999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.871,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.843,"5min":1.8260000000000001,"15min":1.3120000000000001},"min":{"1min":0.50700000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.7960000000000003,"5min":6.7960000000000003,"15min":6.7960000000000003},"last":0.98299999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.4350000000000001,"15min":1.1859999999999999},"min":{"1min":0.39100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.1189999999999998,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":1.143},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.512,"15min":1.258},"min":{"1min":0.45400000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.7720000000000002,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":1.036}]},{"osd":7,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.0249999999999999,"15min":1.4279999999999999},"min":{"1min":0.76700000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":6.7229999999999999,"5min":7.46,"15min":7.46},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.653,"15min":1.3160000000000001},"min":{"1min":0.48599999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":6.9180000000000001,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":1.2270000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122171,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661196,"kb_used_data":494980,"kb_used_omap":8,"kb_used_meta":166199,"kb_avail":93083956,"statfs":{"total":95995035648,"available":95317970944,"internally_reserved":0,"allocated":506859520,"data_stored":506198426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":170188032},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.6890000000000001,"15min":0.97899999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.1750000000000007,"5min":12.460000000000001,"15min":12.460000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.5149999999999999,"15min":0.99199999999999999},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.165,"5min":10.243,"15min":10.243},"last":0.81899999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.532,"15min":0.94999999999999996},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3570000000000002,"5min":13.035,"15min":13.035},"last":0.80200000000000005},{"interface":"front","average":{"1min":2.024,"5min":1.738,"15min":1.0640000000000001},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.006,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.874}]},{"osd":3,"last update":"Mon Oct 9 06:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.288,"15min":0.92800000000000005},"min":{"1min":0.255,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.0700000000000001,"15min":0.86799999999999999},"min":{"1min":0.35699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.2549999999999999,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":0.96099999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.8320000000000001,"15min":1.228},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.8729999999999993,"5min":9.0890000000000004,"15min":9.0890000000000004},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.446,"5min":1.952,"15min":1.3120000000000001},"min":{"1min":0.48099999999999998,"5min":0.37,"15min":0.37},"max":{"1min":7.9249999999999998,"5min":10.505000000000001,"15min":10.505000000000001},"last":1.1200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.181,"15min":1.4119999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.5339999999999998,"5min":9.548,"15min":9.548},"last":1.032},{"interface":"front","average":{"1min":1.7729999999999999,"5min":1.8100000000000001,"15min":1.302},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.329000000000001,"5min":10.329000000000001,"15min":10.329000000000001},"last":1.202}]},{"osd":6,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.5189999999999999,"15min":1.218},"min":{"1min":0.45500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.9169999999999998,"5min":10.194000000000001,"15min":10.194000000000001},"last":1.3440000000000001},{"interface":"front","average":{"1min":1.411,"5min":1.554,"15min":1.1910000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.7629999999999999,"5min":8.7629999999999999,"15min":8.7629999999999999},"last":1.093}]},{"osd":7,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.6719999999999999,"15min":1.321},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.0330000000000004,"5min":6.0330000000000004,"15min":6.0330000000000004},"last":1.3009999999999999},{"interface":"front","average":{"1min":2.1899999999999999,"5min":2.0059999999999998,"15min":1.409},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":8.7959999999999994,"5min":8.7959999999999994,"15min":8.7959999999999994},"last":1.28}]}]},{"osd":2,"up_from":98,"seq":420906795120,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":131592,"kb_used_data":5632,"kb_used_omap":11,"kb_used_meta":125940,"kb_avail":93613560,"statfs":{"total":95995035648,"available":95860285440,"internally_reserved":0,"allocated":5767168,"data_stored":5113636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":128963518},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":2.0150000000000001,"15min":1.135},"min":{"1min":0.224,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.0839999999999996,"5min":35.015999999999998,"15min":35.015999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.306,"5min":2.3719999999999999,"15min":1.2290000000000001},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":4.492,"5min":36.415999999999997,"15min":36.415999999999997},"last":0.51400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.29699999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.6050000000000004,"5min":8.298,"15min":8.298},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.9099999999999999,"15min":1.113},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.0350000000000001,"5min":10.4,"15min":10.4},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.371,"15min":0.94899999999999995},"min":{"1min":0.35299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.311999999999999,"5min":13.311999999999999,"15min":13.311999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.2190000000000001,"15min":0.875},"min":{"1min":0.35599999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.306,"5min":18.831,"15min":18.831},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":2.5899999999999999,"15min":1.502},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":5.7439999999999998,"5min":19.184000000000001,"15min":19.184000000000001},"last":1.026},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.2090000000000001,"15min":1.363},"min":{"1min":0.41699999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":29.408000000000001,"5min":29.408000000000001,"15min":29.408000000000001},"last":1.0740000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.2320000000000002,"15min":1.417},"min":{"1min":0.39700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":8.0830000000000002,"5min":12.833,"15min":12.833},"last":1.135},{"interface":"front","average":{"1min":3.0449999999999999,"5min":1.9770000000000001,"15min":1.2949999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":29.545000000000002,"5min":29.545000000000002,"15min":29.545000000000002},"last":1.163}]},{"osd":6,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":1.5569999999999999,"15min":1.1910000000000001},"min":{"1min":0.374,"5min":0.372,"15min":0.372},"max":{"1min":29.504000000000001,"5min":29.504000000000001,"15min":29.504000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.986,"5min":1.698,"15min":1.234},"min":{"1min":0.46899999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.6760000000000002,"5min":8.1349999999999998,"15min":8.1349999999999998},"last":0.97299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.827,"15min":1.3380000000000001},"min":{"1min":0.52200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.8630000000000004,"5min":8.3309999999999995,"15min":8.3309999999999995},"last":1.105},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.2799999999999998,"15min":1.494},"min":{"1min":0.435,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":4.585,"5min":10.978999999999999,"15min":10.978999999999999},"last":0.88200000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631594,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77308,"kb_used_data":5620,"kb_used_omap":3,"kb_used_meta":71676,"kb_avail":93667844,"statfs":{"total":95995035648,"available":95915872256,"internally_reserved":0,"allocated":5754880,"data_stored":5112193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":73396769},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.978,"15min":1.105},"min":{"1min":0.215,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":1.6399999999999999,"5min":62.939,"15min":62.939},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.1309999999999998,"15min":1.1459999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":4.7590000000000003,"5min":63.869,"15min":63.869},"last":0.52400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.0089999999999999,"15min":0.80900000000000005},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.8570000000000002,"5min":5.944,"15min":5.944},"last":0.378},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.0580000000000001,"15min":0.86399999999999999},"min":{"1min":0.40500000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.4329999999999998,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.2490000000000001,"15min":1.294},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.78,"5min":70.594999999999999,"15min":70.594999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.002,"5min":1.859,"15min":1.113},"min":{"1min":0.29199999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.8339999999999996,"5min":62.991999999999997,"15min":62.991999999999997},"last":0.48499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.804,"15min":1.2170000000000001},"min":{"1min":0.42199999999999999,"5min":0.314,"15min":0.314},"max":{"1min":4.7130000000000001,"5min":10.446999999999999,"15min":10.446999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.613,"5min":1.667,"15min":1.218},"min":{"1min":0.43099999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.5830000000000002,"5min":7.8940000000000001,"15min":7.8940000000000001},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.446,"15min":1.171},"min":{"1min":0.46500000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.0679999999999996,"5min":6.0910000000000002,"15min":6.0910000000000002},"last":0.79600000000000004},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.7669999999999999,"15min":1.3029999999999999},"min":{"1min":0.40400000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7370000000000001,"5min":11.754,"15min":11.754},"last":0.81200000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":1.1499999999999999,"15min":1.127},"min":{"1min":0.39000000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.6360000000000001,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.3660000000000001,"15min":1.1839999999999999},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.1150000000000002,"5min":6.0250000000000004,"15min":6.0250000000000004},"last":0.75}]},{"osd":7,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.4790000000000001,"15min":1.202},"min":{"1min":0.55000000000000004,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.6669999999999998,"5min":6.4820000000000002,"15min":6.4820000000000002},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.512,"15min":1.262},"min":{"1min":0.46000000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.681,"5min":11.439,"15min":11.439},"last":0.66000000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019202,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":545324,"kb_used_data":231716,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":93199828,"statfs":{"total":95995035648,"available":95436623872,"internally_reserved":0,"allocated":237277184,"data_stored":236759724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.6240000000000001,"15min":1.859},"min":{"1min":0.32500000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.3949999999999996,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.74299999999999999},{"interface":"front","average":{"1min":2.5950000000000002,"5min":3.9300000000000002,"15min":1.994},"min":{"1min":0.39500000000000002,"5min":0.312,"15min":0.312},"max":{"1min":7.8049999999999997,"5min":84.912999999999997,"15min":84.912999999999997},"last":0.75900000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":2.3559999999999999,"15min":1.4319999999999999},"min":{"1min":0.38100000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.919,"5min":61.726999999999997,"15min":61.726999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.5449999999999999,"5min":3.0539999999999998,"15min":1.677},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.265000000000001,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.52800000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2519999999999998,"5min":3.3559999999999999,"15min":1.726},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.9489999999999998,"5min":61.783999999999999,"15min":61.783999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":2.9350000000000001,"5min":3.5550000000000002,"15min":1.7789999999999999},"min":{"1min":0.36699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.56,"5min":61.551000000000002,"15min":61.551000000000002},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.29,"15min":1.448},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.0979999999999999,"5min":36.154000000000003,"15min":36.154000000000003},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.6890000000000001,"15min":1.254},"min":{"1min":0.40600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":9.3490000000000002,"5min":36.064,"15min":36.064},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:23:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.8109999999999999,"15min":1.2230000000000001},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.0030000000000001,"5min":12.242000000000001,"15min":12.242000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.77100000000000002,"5min":2.5129999999999999,"15min":1.4930000000000001},"min":{"1min":0.34100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.2470000000000001,"5min":20.113,"15min":20.113},"last":0.68200000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:22:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.1749999999999998,"15min":1.413},"min":{"1min":0.374,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.8529999999999998,"5min":18.010000000000002,"15min":18.010000000000002},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.677,"15min":1.2410000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.867,"5min":13.913,"15min":13.913},"last":0.78800000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:22:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.2850000000000001,"15min":1.478},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":7.8840000000000003,"5min":15.294,"15min":15.294},"last":0.83299999999999996},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.3679999999999999,"15min":1.5009999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.731999999999999,"5min":10.731999999999999,"15min":10.731999999999999},"last":0.85099999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822969,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302688,"kb_used_data":1368,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442464,"statfs":{"total":95995035648,"available":95685083136,"internally_reserved":0,"allocated":1400832,"data_stored":898530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":3.5710000000000002,"15min":1.8149999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":5.6719999999999997,"5min":69.793999999999997,"15min":69.793999999999997},"last":0.747},{"interface":"front","average":{"1min":1.774,"5min":2.9969999999999999,"15min":1.5980000000000001},"min":{"1min":0.35599999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.0090000000000003,"5min":68.873000000000005,"15min":68.873000000000005},"last":0.52000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.5619999999999998,"15min":1.476},"min":{"1min":0.372,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3409999999999993,"5min":52.070999999999998,"15min":52.070999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.363,"5min":2.8570000000000002,"15min":1.5580000000000001},"min":{"1min":0.48999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.8700000000000001,"5min":51.570999999999998,"15min":51.570999999999998},"last":0.79600000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":4.5620000000000003,"5min":3.1120000000000001,"15min":1.657},"min":{"1min":0.313,"5min":0.307,"15min":0.307},"max":{"1min":45.466000000000001,"5min":47.207000000000001,"15min":47.207000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":4.6390000000000002,"5min":3.1419999999999999,"15min":1.728},"min":{"1min":0.39000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":59.082999999999998,"5min":59.082999999999998,"15min":59.082999999999998},"last":0.57599999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:23:00 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.851,"15min":1.292},"min":{"1min":0.40200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2439999999999998,"5min":12.334,"15min":12.334},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.361,"5min":1.6259999999999999,"15min":1.2390000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.4119999999999999,"5min":11.823,"15min":11.823},"last":0.626}]},{"osd":4,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.6709999999999998,"15min":1.494},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.669,"5min":17.667999999999999,"15min":17.667999999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.468,"5min":2.202,"15min":1.4119999999999999},"min":{"1min":0.52400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.9720000000000004,"5min":16.181999999999999,"15min":16.181999999999999},"last":1.0269999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.1110000000000002,"15min":1.361},"min":{"1min":0.495,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.9139999999999997,"5min":17.454999999999998,"15min":17.454999999999998},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.677,"5min":2.0840000000000001,"15min":1.4330000000000001},"min":{"1min":0.58499999999999996,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":9.2919999999999998,"5min":12.365,"15min":12.365},"last":0.95299999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.379,"5min":2.3540000000000001,"15min":1.5289999999999999},"min":{"1min":0.495,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.3170000000000002,"5min":13.047000000000001,"15min":13.047000000000001},"last":1.1890000000000001},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.2130000000000001,"15min":1.4870000000000001},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":10.465999999999999,"5min":11.981,"15min":11.981},"last":0.70699999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:23:25.812 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:25.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:25.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:25.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:25.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:26.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:25 smithi155 bash[59495]: audit 2023-10-09T06:23:24.363393+0000 mgr.y (mgr.25723) 546 : audit 0 from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:25 smithi155 bash[59495]: cluster 2023-10-09T06:23:24.884878+0000 mgr.y (mgr.25723) 547 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-09T06:23:26.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:26.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:26.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5732 5716 178.612 166.25 0.00630743 0.00558305 2023-10-09T06:23:26.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:26.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:26.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:26.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:26.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:25 smithi125 bash[77299]: audit 2023-10-09T06:23:24.363393+0000 mgr.y (mgr.25723) 546 : audit 0 from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:26.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:25 smithi125 bash[77299]: cluster 2023-10-09T06:23:24.884878+0000 mgr.y (mgr.25723) 547 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-09T06:23:26.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:25 smithi125 bash[80002]: audit 2023-10-09T06:23:24.363393+0000 mgr.y (mgr.25723) 546 : audit 0 from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:26.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:25 smithi125 bash[80002]: cluster 2023-10-09T06:23:24.884878+0000 mgr.y (mgr.25723) 547 : cluster 0 pgmap v518: 89 pgs: 89 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-09T06:23:26.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:26.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:26.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:26.626 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:26.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:26.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:26.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:26.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:26.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:26.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:27.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:27.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:27.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8270 8254 171.944 158.625 0.00583406 0.00580687 2023-10-09T06:23:27.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:27.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:27.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:27.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:27.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:27.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:27.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:27.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:27.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:27.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:27.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:27.940 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:27 smithi125 bash[77299]: cluster 2023-10-09T06:23:26.885665+0000 mgr.y (mgr.25723) 548 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s 2023-10-09T06:23:27.941 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:27 smithi125 bash[80002]: cluster 2023-10-09T06:23:26.885665+0000 mgr.y (mgr.25723) 548 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s 2023-10-09T06:23:27.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:27.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:28.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:28.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:27 smithi155 bash[59495]: cluster 2023-10-09T06:23:26.885665+0000 mgr.y (mgr.25723) 548 : cluster 0 pgmap v519: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s wr, 320 op/s 2023-10-09T06:23:28.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 10862 10846 169.455 162 0.00565583 0.00589316 2023-10-09T06:23:28.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:28.313 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:28.315 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:28.319 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:28.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:28.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:28.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:28.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:28.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:28.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:28.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:28.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:28.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:28.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:23:28.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:28.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:29.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:29.029 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:29.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:29.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 13109 13093 163.647 140.438 0.008308 0.00610438 2023-10-09T06:23:29.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:29.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:29.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:29.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:29.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:29.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:29.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:29.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:29.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:29.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:29 smithi125 bash[77299]: cluster 2023-10-09T06:23:28.886216+0000 mgr.y (mgr.25723) 549 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s 2023-10-09T06:23:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:29 smithi125 bash[80002]: cluster 2023-10-09T06:23:28.886216+0000 mgr.y (mgr.25723) 549 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s 2023-10-09T06:23:29.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.934 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:29 smithi155 bash[59495]: cluster 2023-10-09T06:23:28.886216+0000 mgr.y (mgr.25723) 549 : cluster 0 pgmap v520: 89 pgs: 89 active+clean; 249 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s 2023-10-09T06:23:29.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:29.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:30.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:30.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:30.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:30.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 15543 15527 161.724 152.125 0.00594331 0.00617522 2023-10-09T06:23:30.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:30.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:30.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:30.354 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:30.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:30.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:30.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:30.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:30.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:30.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:30.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:30.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:30.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:30.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:31.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:31.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:31.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:31.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 17783 17767 158.619 140 0.00524992 0.00629754 2023-10-09T06:23:31.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:31.284 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:31.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:31.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:31] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-10-09T06:23:31.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:31.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:31.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:31.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:31.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:31.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:31.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:31.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:31.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:31.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:31.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:32.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:32.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:31 smithi155 bash[59495]: cluster 2023-10-09T06:23:30.886931+0000 mgr.y (mgr.25723) 550 : cluster 0 pgmap v521: 89 pgs: 89 active+clean; 798 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:23:32.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 19870 19854 155.094 130.438 0.00263135 0.0064031 2023-10-09T06:23:32.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:32.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:32.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:31 smithi125 bash[77299]: cluster 2023-10-09T06:23:30.886931+0000 mgr.y (mgr.25723) 550 : cluster 0 pgmap v521: 89 pgs: 89 active+clean; 798 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:23:32.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:31 smithi125 bash[80002]: cluster 2023-10-09T06:23:30.886931+0000 mgr.y (mgr.25723) 550 : cluster 0 pgmap v521: 89 pgs: 89 active+clean; 798 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:23:32.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:32.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:32.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:32.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:32.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:32.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:32.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:32.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:32.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:32.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:32.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:33.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:33.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:33.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 21587 21571 149.784 107.312 0.0115901 0.00666983 2023-10-09T06:23:33.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:33.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:33.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:33.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:33.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:33.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:33.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:33.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.659 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:33.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:33.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:33.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:33.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:34.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:34.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[77299]: cluster 2023-10-09T06:23:32.887452+0000 mgr.y (mgr.25723) 551 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 940 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:23:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[77299]: audit 2023-10-09T06:23:33.580745+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:34.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[77299]: audit 2023-10-09T06:23:33.581957+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[80002]: cluster 2023-10-09T06:23:32.887452+0000 mgr.y (mgr.25723) 551 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 940 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:23:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[80002]: audit 2023-10-09T06:23:33.580745+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:33 smithi125 bash[80002]: audit 2023-10-09T06:23:33.581957+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:34.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 23634 23618 147.598 127.938 0.00313951 0.00676663 2023-10-09T06:23:34.177 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:34.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:34.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:34.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:34.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:34.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:33 smithi155 bash[59495]: cluster 2023-10-09T06:23:32.887452+0000 mgr.y (mgr.25723) 551 : cluster 0 pgmap v522: 89 pgs: 89 active+clean; 940 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:23:34.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:33 smithi155 bash[59495]: audit 2023-10-09T06:23:33.580745+0000 mon.a (mon.0) 761 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:23:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:33 smithi155 bash[59495]: audit 2023-10-09T06:23:33.581957+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:34.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:34.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:34.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:34.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:34.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:34.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:34.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:34.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:34.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:34.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:34.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:35.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:35.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:35.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:35.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 25554 25538 145.088 120 0.00739615 0.00688641 2023-10-09T06:23:35.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:35.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:35.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:35.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:35.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:35.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:35.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:35.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:35.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:35.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:35.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:35.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:35.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:35.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:36.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:36.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:36.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:35 smithi125 bash[77299]: cluster 2023-10-09T06:23:34.888205+0000 mgr.y (mgr.25723) 552 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:23:36.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:35 smithi125 bash[80002]: cluster 2023-10-09T06:23:34.888205+0000 mgr.y (mgr.25723) 552 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:23:36.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:36.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 27517 27501 143.221 122.688 0.00944505 0.00697703 2023-10-09T06:23:36.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:36.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:36.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:36.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:35 smithi155 bash[59495]: cluster 2023-10-09T06:23:34.888205+0000 mgr.y (mgr.25723) 552 : cluster 0 pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:23:36.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:36.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:36.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:36.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:36.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:36.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:36.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:36.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:36.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:36.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:36.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:37.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:37.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:37.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 29226 29210 140.419 106.812 0.0250793 0.00711245 2023-10-09T06:23:37.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:37.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:37.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:37.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:37.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:37.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:37.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:37.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:37.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:37.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:37.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:37.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:37.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:38.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:38.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:38.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:37 smithi155 bash[59495]: cluster 2023-10-09T06:23:36.888849+0000 mgr.y (mgr.25723) 553 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:23:38.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 15 30756 30741 137.224 95.6875 0.014803 0.00728083 2023-10-09T06:23:38.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:38.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:38.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:38.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:38.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:38 smithi125 bash[77299]: cluster 2023-10-09T06:23:36.888849+0000 mgr.y (mgr.25723) 553 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:23:38.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:38 smithi125 bash[80002]: cluster 2023-10-09T06:23:36.888849+0000 mgr.y (mgr.25723) 553 : cluster 0 pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:23:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:38.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:38.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:38.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:38.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:38.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:38.726 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:38.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:38.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:38.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:38.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:38.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:39.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:39.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:39.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 32383 32367 134.85 101.625 0.0107787 0.00740891 2023-10-09T06:23:39.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:39.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:39.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:39.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:39.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:39.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:39.763 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:39.783 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:39 smithi155 bash[59495]: cluster 2023-10-09T06:23:38.889422+0000 mgr.y (mgr.25723) 554 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:23:39.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:39.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:39.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:39.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:40.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:40.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:40.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:39 smithi125 bash[77299]: cluster 2023-10-09T06:23:38.889422+0000 mgr.y (mgr.25723) 554 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:23:40.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:39 smithi125 bash[80002]: cluster 2023-10-09T06:23:38.889422+0000 mgr.y (mgr.25723) 554 : cluster 0 pgmap v525: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 123 MiB/s wr, 1.98k op/s 2023-10-09T06:23:40.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 33875 33859 132.249 93.25 0.00618682 0.00755613 2023-10-09T06:23:40.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:40.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:40.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:40.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:40.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:40.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:40.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:40.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:40.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:40.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:40.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:40.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:40.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:40.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:40.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:40.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:40.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:41.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:41.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:41.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:41.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 35340 35324 129.855 91.5625 0.00459438 0.00769502 2023-10-09T06:23:41.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:41.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:41.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:41.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:41.394 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:41] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.33.4" 2023-10-09T06:23:41.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:41.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:41.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:41.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:41.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:41.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:41.817 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:41.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:41.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:41.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:42.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:42.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:41 smithi125 bash[77299]: cluster 2023-10-09T06:23:40.890395+0000 mgr.y (mgr.25723) 555 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T06:23:42.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:41 smithi125 bash[80002]: cluster 2023-10-09T06:23:40.890395+0000 mgr.y (mgr.25723) 555 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T06:23:42.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 36990 36974 128.37 103.125 0.00770305 0.00778424 2023-10-09T06:23:42.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:42.203 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:41 smithi155 bash[59495]: cluster 2023-10-09T06:23:40.890395+0000 mgr.y (mgr.25723) 555 : cluster 0 pgmap v526: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-10-09T06:23:42.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:42.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:42.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:42.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:42.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:42.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:42.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:42.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:42.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:42.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:42.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:42.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:43.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:43.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:43.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:43.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 38245 38229 125.741 78.4375 0.00303011 0.00794734 2023-10-09T06:23:43.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:43.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:43.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:43.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:43.425 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:43.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:43.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:43.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:43.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:43.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:43.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:43.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:43.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:43.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:43.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:44.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:44.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:44.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:23:44.160708+0000 min lat: 0.00129902 max lat: 0.206454 avg lat: 0.00801449 2023-10-09T06:23:44.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:23:44.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 39525 39509 123.454 80 0.00239282 0.00801449 2023-10-09T06:23:44.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:44.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:43 smithi125 bash[77299]: cluster 2023-10-09T06:23:42.890891+0000 mgr.y (mgr.25723) 556 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T06:23:44.252 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:43 smithi125 bash[80002]: cluster 2023-10-09T06:23:42.890891+0000 mgr.y (mgr.25723) 556 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T06:23:44.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:44.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:43 smithi155 bash[59495]: cluster 2023-10-09T06:23:42.890891+0000 mgr.y (mgr.25723) 556 : cluster 0 pgmap v527: 89 pgs: 89 active+clean; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-09T06:23:44.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:44.589 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:44.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:44.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:44.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:44.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:44.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:44.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:45.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:45.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:45.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:45.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 40900 40884 121.667 85.9375 0.00725837 0.00821307 2023-10-09T06:23:45.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:45.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:45.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:45.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:45.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:45.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:45.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:45.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:45.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:45.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:45.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:45.934 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:45.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:45.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:46.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:46.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:45 smithi155 bash[59495]: cluster 2023-10-09T06:23:44.892330+0000 mgr.y (mgr.25723) 557 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T06:23:46.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 42123 42107 119.611 76.4375 0.00631296 0.00835294 2023-10-09T06:23:46.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:46.212 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:45 smithi125 bash[77299]: cluster 2023-10-09T06:23:44.892330+0000 mgr.y (mgr.25723) 557 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T06:23:46.212 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:45 smithi125 bash[80002]: cluster 2023-10-09T06:23:44.892330+0000 mgr.y (mgr.25723) 557 : cluster 0 pgmap v528: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-09T06:23:46.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:46.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:46.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:46.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:46.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:46.742 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:46.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:46.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:46.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:46.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:47.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:47.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:47.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 43429 43413 117.959 81.625 0.00275143 0.00846337 2023-10-09T06:23:47.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:47.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:47.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:47.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:47.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:47.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:47.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:47.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:47.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:47.780 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:23:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:23:47.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:44089 6.9 6:91ed2725:::benchmark_data_smithi125_130224_object44088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:23:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:47.769599+0000", 2023-10-09T06:23:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0039419590000000001, 2023-10-09T06:23:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0039762640000000002, 2023-10-09T06:23:47.781 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:23:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:23:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:23:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:23:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:23:47.782 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 44089 2023-10-09T06:23:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:23:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:23:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769599+0000", 2023-10-09T06:23:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:23:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769599+0000", 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:23:47.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769597+0000", 2023-10-09T06:23:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:23:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769674+0000", 2023-10-09T06:23:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6725999999999997e-05 2023-10-09T06:23:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:23:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769691+0000", 2023-10-09T06:23:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6520000000000001e-05 2023-10-09T06:23:47.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:23:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769699+0000", 2023-10-09T06:23:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0369999999999995e-06 2023-10-09T06:23:47.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:23:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769848+0000", 2023-10-09T06:23:47.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014967999999999999 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.769985+0000", 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136277 2023-10-09T06:23:47.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:23:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.770125+0000", 2023-10-09T06:23:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140667 2023-10-09T06:23:47.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:23:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.771836+0000", 2023-10-09T06:23:47.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017108760000000001 2023-10-09T06:23:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:47.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772176+0000", 2023-10-09T06:23:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000339599 2023-10-09T06:23:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:23:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:23:47.795 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:23:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.796 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:44095 6.0 6:01dd4341:::benchmark_data_smithi125_130224_object44094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:23:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:47.772350+0000", 2023-10-09T06:23:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001190331, 2023-10-09T06:23:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013243529999999999, 2023-10-09T06:23:47.797 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 44095 2023-10-09T06:23:47.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:23:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:23:47.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772350+0000", 2023-10-09T06:23:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:23:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:23:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772350+0000", 2023-10-09T06:23:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:23:47.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:23:47.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772349+0000", 2023-10-09T06:23:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:23:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:23:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772367+0000", 2023-10-09T06:23:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7136999999999999e-05 2023-10-09T06:23:47.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:23:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772377+0000", 2023-10-09T06:23:47.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0689e-05 2023-10-09T06:23:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:23:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772383+0000", 2023-10-09T06:23:47.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7989999999999999e-06 2023-10-09T06:23:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.808 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:23:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772422+0000", 2023-10-09T06:23:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8445000000000002e-05 2023-10-09T06:23:47.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:23:47.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772529+0000", 2023-10-09T06:23:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107354 2023-10-09T06:23:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:23:47.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:23:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:23:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:47.772635+0000", 2023-10-09T06:23:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106464 2023-10-09T06:23:47.812 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:23:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:23:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:23:47.813 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:23:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T06:23:47.814 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 2 2023-10-09T06:23:47.814 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:23:47.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:47.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:47.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:47.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:47.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:48.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:48.082 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:47 smithi125 bash[77299]: cluster 2023-10-09T06:23:46.893120+0000 mgr.y (mgr.25723) 558 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T06:23:48.083 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:47 smithi125 bash[80002]: cluster 2023-10-09T06:23:46.893120+0000 mgr.y (mgr.25723) 558 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T06:23:48.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:47 smithi155 bash[59495]: cluster 2023-10-09T06:23:46.893120+0000 mgr.y (mgr.25723) 558 : cluster 0 pgmap v529: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T06:23:48.166 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 44660 44644 116.249 76.9375 0.00585588 0.00859668 2023-10-09T06:23:48.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:48.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:48.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.354 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:48.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:48.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:48.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.634 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:23:48.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:48.671 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:23:48.723 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:23:48.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:48.762 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:23:48.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:48.798 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:23:48.837 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:23:48.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.875 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:23:48.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:48.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:48.910 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.916 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:23:49.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:49.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:48 smithi155 bash[59495]: audit 2023-10-09T06:23:48.571968+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:49.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:48 smithi155 bash[59495]: audit 2023-10-09T06:23:48.583988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.125:0/4153222703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:49.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 45944 45928 114.803 80.25 0.0214947 0.00870228 2023-10-09T06:23:49.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:49.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:49.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:49.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:48 smithi125 bash[77299]: audit 2023-10-09T06:23:48.571968+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:49.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:48 smithi125 bash[77299]: audit 2023-10-09T06:23:48.583988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.125:0/4153222703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:48 smithi125 bash[80002]: audit 2023-10-09T06:23:48.571968+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:48 smithi125 bash[80002]: audit 2023-10-09T06:23:48.583988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.125:0/4153222703' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:49.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:49.558 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:23:49.566 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:23:49.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:49.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:49.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:49.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:49.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:49.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:49.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:49.974 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:49 smithi125 bash[80002]: cluster 2023-10-09T06:23:48.893602+0000 mgr.y (mgr.25723) 559 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:23:49.974 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:49 smithi125 bash[77299]: cluster 2023-10-09T06:23:48.893602+0000 mgr.y (mgr.25723) 559 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:23:49.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:49.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:49.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:50.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.108 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:49 smithi155 bash[59495]: cluster 2023-10-09T06:23:48.893602+0000 mgr.y (mgr.25723) 559 : cluster 0 pgmap v530: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:23:50.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:50.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 15 47354 47339 113.779 88.1875 0.007181 0.00878363 2023-10-09T06:23:50.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:50.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:50.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:50.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:50.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:50.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:50.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:50.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:50.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:50.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:50.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:51.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:51.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:51.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:50 smithi125 bash[77299]: audit 2023-10-09T06:23:49.532167+0000 mgr.y (mgr.25723) 560 : audit 0 from='client.35070 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:51.057 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:50 smithi125 bash[80002]: audit 2023-10-09T06:23:49.532167+0000 mgr.y (mgr.25723) 560 : audit 0 from='client.35070 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:51.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:51.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 48638 48622 112.535 80.1875 0.00526423 0.00888217 2023-10-09T06:23:51.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:51.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:51.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:50 smithi155 bash[59495]: audit 2023-10-09T06:23:49.532167+0000 mgr.y (mgr.25723) 560 : audit 0 from='client.35070 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:51.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:51.393 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:23:51] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-10-09T06:23:51.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.434 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:51.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:51.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:51.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:51.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:23:51.799 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:51.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:51.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:51.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:51.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[77299]: cluster 2023-10-09T06:23:50.894299+0000 mgr.y (mgr.25723) 561 : cluster 0 pgmap v531: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:23:51.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[77299]: audit 2023-10-09T06:23:51.222267+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.125:0/1768806886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:51.986 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[77299]: audit 2023-10-09T06:23:51.223126+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:52.013 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:23:52.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:52.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:51 smithi155 bash[59495]: cluster 2023-10-09T06:23:50.894299+0000 mgr.y (mgr.25723) 561 : cluster 0 pgmap v531: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:23:52.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:51 smithi155 bash[59495]: audit 2023-10-09T06:23:51.222267+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.125:0/1768806886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:52.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:51 smithi155 bash[59495]: audit 2023-10-09T06:23:51.223126+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:52.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:52.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 49926 49910 111.39 80.5 0.0063961 0.008957 2023-10-09T06:23:52.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:52.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:52.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:52.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:52.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:52.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[80002]: cluster 2023-10-09T06:23:50.894299+0000 mgr.y (mgr.25723) 561 : cluster 0 pgmap v531: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:23:52.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[80002]: audit 2023-10-09T06:23:51.222267+0000 mon.c (mon.2) 57 : audit 1 from='client.? 172.21.15.125:0/1768806886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:52.473 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:51 smithi125 bash[80002]: audit 2023-10-09T06:23:51.223126+0000 mon.a (mon.0) 764 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:52.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:52.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:52.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:52.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:52.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:52.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:52.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:53.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:51.975994+0000 mon.a (mon.0) 765 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: audit 2023-10-09T06:23:51.999457+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:51.999767+0000 mon.a (mon.0) 767 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:52.003540+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:52.010568+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:52.016347+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[77299]: cluster 2023-10-09T06:23:52.018195+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:51.975994+0000 mon.a (mon.0) 765 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: audit 2023-10-09T06:23:51.999457+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:51.999767+0000 mon.a (mon.0) 767 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:52.003540+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:52.010568+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:52.016347+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:52 smithi125 bash[80002]: cluster 2023-10-09T06:23:52.018195+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.157 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:53.167 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 15 51127 51112 110.14 75.125 0.0070229 0.00907377 2023-10-09T06:23:53.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:53.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:51.975994+0000 mon.a (mon.0) 765 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: audit 2023-10-09T06:23:51.999457+0000 mon.a (mon.0) 766 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:51.999767+0000 mon.a (mon.0) 767 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:52.003540+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:52.010568+0000 osd.4 (osd.4) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:52.016347+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:52 smithi155 bash[59495]: cluster 2023-10-09T06:23:52.018195+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e147 with expected crc 2023-10-09T06:23:53.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:53.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:53.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:53.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:53.492 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":530,"stamp":"2023-10-09T06:23:48.893315+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2717587379,"num_objects":41422,"num_object_clones":0,"num_object_copies":124266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41422,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":42500,"num_write_kb":2664375,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":1190420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42564,"ondisk_log_size":42564,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10198704,"kb_used_data":8413232,"kb_used_omap":30,"kb_used_meta":1785313,"kb_avail":739762512,"statfs":{"total":767960285184,"available":757516812288,"internally_reserved":0,"allocated":8615149568,"data_stored":8610464761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1828160623},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,5,8,0,0,1,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1065549824,"num_objects":16259,"num_object_clones":0,"num_object_copies":48777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16259,"num_write_kb":1040576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004926"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":127,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410111+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:23:19.410111+0000","last_peered":"2023-10-09T06:23:19.410111+0000","last_clean":"2023-10-09T06:23:19.410111+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:23:19.410111+0000","last_undegraded":"2023-10-09T06:23:19.410111+0000","last_fullsized":"2023-10-09T06:23:19.410111+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.879582+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:23:18.879582+0000","last_peered":"2023-10-09T06:23:18.879582+0000","last_clean":"2023-10-09T06:23:18.879582+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:23:18.879582+0000","last_undegraded":"2023-10-09T06:23:18.879582+0000","last_fullsized":"2023-10-09T06:23:18.879582+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.371588+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:23:21.371588+0000","last_peered":"2023-10-09T06:23:21.371588+0000","last_clean":"2023-10-09T06:23:21.371588+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:23:21.371588+0000","last_undegraded":"2023-10-09T06:23:21.371588+0000","last_fullsized":"2023-10-09T06:23:21.371588+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.402051+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:23:19.402051+0000","last_peered":"2023-10-09T06:23:19.402051+0000","last_clean":"2023-10-09T06:23:19.402051+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:23:19.402051+0000","last_undegraded":"2023-10-09T06:23:19.402051+0000","last_fullsized":"2023-10-09T06:23:19.402051+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878533+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:23:18.878533+0000","last_peered":"2023-10-09T06:23:18.878533+0000","last_clean":"2023-10-09T06:23:18.878533+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:23:18.878533+0000","last_undegraded":"2023-10-09T06:23:18.878533+0000","last_fullsized":"2023-10-09T06:23:18.878533+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410626+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:23:19.410626+0000","last_peered":"2023-10-09T06:23:19.410626+0000","last_clean":"2023-10-09T06:23:19.410626+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:23:19.410626+0000","last_undegraded":"2023-10-09T06:23:19.410626+0000","last_fullsized":"2023-10-09T06:23:19.410626+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401627+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:23:19.401627+0000","last_peered":"2023-10-09T06:23:19.401627+0000","last_clean":"2023-10-09T06:23:19.401627+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:23:19.401627+0000","last_undegraded":"2023-10-09T06:23:19.401627+0000","last_fullsized":"2023-10-09T06:23:19.401627+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878529+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:23:18.878529+0000","last_peered":"2023-10-09T06:23:18.878529+0000","last_clean":"2023-10-09T06:23:18.878529+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:23:18.878529+0000","last_undegraded":"2023-10-09T06:23:18.878529+0000","last_fullsized":"2023-10-09T06:23:18.878529+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900733+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:23:18.900733+0000","last_peered":"2023-10-09T06:23:18.900733+0000","last_clean":"2023-10-09T06:23:18.900733+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:23:18.900733+0000","last_undegraded":"2023-10-09T06:23:18.900733+0000","last_fullsized":"2023-10-09T06:23:18.900733+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.880992+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:23:18.880992+0000","last_peered":"2023-10-09T06:23:18.880992+0000","last_clean":"2023-10-09T06:23:18.880992+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:23:18.880992+0000","last_undegraded":"2023-10-09T06:23:18.880992+0000","last_fullsized":"2023-10-09T06:23:18.880992+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":44,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.328490+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:23:19.328490+0000","last_peered":"2023-10-09T06:23:19.328490+0000","last_clean":"2023-10-09T06:23:19.328490+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:23:19.328490+0000","last_undegraded":"2023-10-09T06:23:19.328490+0000","last_fullsized":"2023-10-09T06:23:19.328490+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400999+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:23:19.400999+0000","last_peered":"2023-10-09T06:23:19.400999+0000","last_clean":"2023-10-09T06:23:19.400999+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:23:19.400999+0000","last_undegraded":"2023-10-09T06:23:19.400999+0000","last_fullsized":"2023-10-09T06:23:19.400999+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":156,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409957+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:23:19.409957+0000","last_peered":"2023-10-09T06:23:19.409957+0000","last_clean":"2023-10-09T06:23:19.409957+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:23:19.409957+0000","last_undegraded":"2023-10-09T06:23:19.409957+0000","last_fullsized":"2023-10-09T06:23:19.409957+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410546+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:23:19.410546+0000","last_peered":"2023-10-09T06:23:19.410546+0000","last_clean":"2023-10-09T06:23:19.410546+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:23:19.410546+0000","last_undegraded":"2023-10-09T06:23:19.410546+0000","last_fullsized":"2023-10-09T06:23:19.410546+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":44,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.328329+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:23:19.328329+0000","last_peered":"2023-10-09T06:23:19.328329+0000","last_clean":"2023-10-09T06:23:19.328329+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:23:19.328329+0000","last_undegraded":"2023-10-09T06:23:19.328329+0000","last_fullsized":"2023-10-09T06:23:19.328329+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":135,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.880448+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:23:18.880448+0000","last_peered":"2023-10-09T06:23:18.880448+0000","last_clean":"2023-10-09T06:23:18.880448+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:23:18.880448+0000","last_undegraded":"2023-10-09T06:23:18.880448+0000","last_fullsized":"2023-10-09T06:23:18.880448+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":115,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401329+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:23:19.401329+0000","last_peered":"2023-10-09T06:23:19.401329+0000","last_clean":"2023-10-09T06:23:19.401329+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:23:19.401329+0000","last_undegraded":"2023-10-09T06:23:19.401329+0000","last_fullsized":"2023-10-09T06:23:19.401329+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900620+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:23:18.900620+0000","last_peered":"2023-10-09T06:23:18.900620+0000","last_clean":"2023-10-09T06:23:18.900620+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:23:18.900620+0000","last_undegraded":"2023-10-09T06:23:18.900620+0000","last_fullsized":"2023-10-09T06:23:18.900620+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":38,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.327716+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:23:19.327716+0000","last_peered":"2023-10-09T06:23:19.327716+0000","last_clean":"2023-10-09T06:23:19.327716+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:23:19.327716+0000","last_undegraded":"2023-10-09T06:23:19.327716+0000","last_fullsized":"2023-10-09T06:23:19.327716+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401981+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:23:19.401981+0000","last_peered":"2023-10-09T06:23:19.401981+0000","last_clean":"2023-10-09T06:23:19.401981+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:23:19.401981+0000","last_undegraded":"2023-10-09T06:23:19.401981+0000","last_fullsized":"2023-10-09T06:23:19.401981+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":120,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409615+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:23:19.409615+0000","last_peered":"2023-10-09T06:23:19.409615+0000","last_clean":"2023-10-09T06:23:19.409615+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:23:19.409615+0000","last_undegraded":"2023-10-09T06:23:19.409615+0000","last_fullsized":"2023-10-09T06:23:19.409615+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":148,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.880076+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:23:18.880076+0000","last_peered":"2023-10-09T06:23:18.880076+0000","last_clean":"2023-10-09T06:23:18.880076+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:23:18.880076+0000","last_undegraded":"2023-10-09T06:23:18.880076+0000","last_fullsized":"2023-10-09T06:23:18.880076+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"146'2530","reported_seq":2542,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.419730+0000","last_change":"2023-10-09T06:23:02.818534+0000","last_active":"2023-10-09T06:23:44.419730+0000","last_peered":"2023-10-09T06:23:44.419730+0000","last_clean":"2023-10-09T06:23:44.419730+0000","last_became_active":"2023-10-09T06:23:02.816918+0000","last_became_peered":"2023-10-09T06:23:02.816918+0000","last_unstale":"2023-10-09T06:23:44.419730+0000","last_undegraded":"2023-10-09T06:23:44.419730+0000","last_fullsized":"2023-10-09T06:23:44.419730+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2530,"log_dups_size":0,"ondisk_log_size":2530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2530,"num_write_kb":161920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":42,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.328277+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:23:19.328277+0000","last_peered":"2023-10-09T06:23:19.328277+0000","last_clean":"2023-10-09T06:23:19.328277+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:23:19.328277+0000","last_undegraded":"2023-10-09T06:23:19.328277+0000","last_fullsized":"2023-10-09T06:23:19.328277+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900191+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:23:18.900191+0000","last_peered":"2023-10-09T06:23:18.900191+0000","last_clean":"2023-10-09T06:23:18.900191+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:23:18.900191+0000","last_undegraded":"2023-10-09T06:23:18.900191+0000","last_fullsized":"2023-10-09T06:23:18.900191+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"146'2620","reported_seq":2633,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:45.710728+0000","last_change":"2023-10-09T06:23:02.810028+0000","last_active":"2023-10-09T06:23:45.710728+0000","last_peered":"2023-10-09T06:23:45.710728+0000","last_clean":"2023-10-09T06:23:45.710728+0000","last_became_active":"2023-10-09T06:23:02.809179+0000","last_became_peered":"2023-10-09T06:23:02.809179+0000","last_unstale":"2023-10-09T06:23:45.710728+0000","last_undegraded":"2023-10-09T06:23:45.710728+0000","last_fullsized":"2023-10-09T06:23:45.710728+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2620,"log_dups_size":0,"ondisk_log_size":2620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:49.851184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2620,"num_write_kb":167680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":42,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.329135+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:23:19.329135+0000","last_peered":"2023-10-09T06:23:19.329135+0000","last_clean":"2023-10-09T06:23:19.329135+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:23:19.329135+0000","last_undegraded":"2023-10-09T06:23:19.329135+0000","last_fullsized":"2023-10-09T06:23:19.329135+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.411414+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:23:19.411414+0000","last_peered":"2023-10-09T06:23:19.411414+0000","last_clean":"2023-10-09T06:23:19.411414+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:23:19.411414+0000","last_undegraded":"2023-10-09T06:23:19.411414+0000","last_fullsized":"2023-10-09T06:23:19.411414+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"146'2500","reported_seq":2511,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.433838+0000","last_change":"2023-10-09T06:23:02.818119+0000","last_active":"2023-10-09T06:23:44.433838+0000","last_peered":"2023-10-09T06:23:44.433838+0000","last_clean":"2023-10-09T06:23:44.433838+0000","last_became_active":"2023-10-09T06:23:02.817802+0000","last_became_peered":"2023-10-09T06:23:02.817802+0000","last_unstale":"2023-10-09T06:23:44.433838+0000","last_undegraded":"2023-10-09T06:23:44.433838+0000","last_fullsized":"2023-10-09T06:23:44.433838+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2500,"log_dups_size":0,"ondisk_log_size":2500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2500,"num_write_kb":160000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":42,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.328651+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:23:19.328651+0000","last_peered":"2023-10-09T06:23:19.328651+0000","last_clean":"2023-10-09T06:23:19.328651+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:23:19.328651+0000","last_undegraded":"2023-10-09T06:23:19.328651+0000","last_fullsized":"2023-10-09T06:23:19.328651+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":129,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410268+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:23:19.410268+0000","last_peered":"2023-10-09T06:23:19.410268+0000","last_clean":"2023-10-09T06:23:19.410268+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:23:19.410268+0000","last_undegraded":"2023-10-09T06:23:19.410268+0000","last_fullsized":"2023-10-09T06:23:19.410268+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"146'2776","reported_seq":2789,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:48.192274+0000","last_change":"2023-10-09T06:23:02.807021+0000","last_active":"2023-10-09T06:23:48.192274+0000","last_peered":"2023-10-09T06:23:48.192274+0000","last_clean":"2023-10-09T06:23:48.192274+0000","last_became_active":"2023-10-09T06:23:02.806707+0000","last_became_peered":"2023-10-09T06:23:02.806707+0000","last_unstale":"2023-10-09T06:23:48.192274+0000","last_undegraded":"2023-10-09T06:23:48.192274+0000","last_fullsized":"2023-10-09T06:23:48.192274+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2776,"log_dups_size":0,"ondisk_log_size":2776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:19.835325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181927936,"num_objects":2776,"num_object_clones":0,"num_object_copies":8328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2776,"num_write_kb":177664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900478+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:23:18.900478+0000","last_peered":"2023-10-09T06:23:18.900478+0000","last_clean":"2023-10-09T06:23:18.900478+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:23:18.900478+0000","last_undegraded":"2023-10-09T06:23:18.900478+0000","last_fullsized":"2023-10-09T06:23:18.900478+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.402534+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:23:19.402534+0000","last_peered":"2023-10-09T06:23:19.402534+0000","last_clean":"2023-10-09T06:23:19.402534+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:23:19.402534+0000","last_undegraded":"2023-10-09T06:23:19.402534+0000","last_fullsized":"2023-10-09T06:23:19.402534+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"146'2525","reported_seq":2537,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.416451+0000","last_change":"2023-10-09T06:23:02.820523+0000","last_active":"2023-10-09T06:23:44.416451+0000","last_peered":"2023-10-09T06:23:44.416451+0000","last_clean":"2023-10-09T06:23:44.416451+0000","last_became_active":"2023-10-09T06:23:02.820107+0000","last_became_peered":"2023-10-09T06:23:02.820107+0000","last_unstale":"2023-10-09T06:23:44.416451+0000","last_undegraded":"2023-10-09T06:23:44.416451+0000","last_fullsized":"2023-10-09T06:23:44.416451+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2525,"log_dups_size":0,"ondisk_log_size":2525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2525,"num_write_kb":161600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":117,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890836+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:23:20.890836+0000","last_peered":"2023-10-09T06:23:20.890836+0000","last_clean":"2023-10-09T06:23:20.890836+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:23:20.890836+0000","last_undegraded":"2023-10-09T06:23:20.890836+0000","last_fullsized":"2023-10-09T06:23:20.890836+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409833+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:23:19.409833+0000","last_peered":"2023-10-09T06:23:19.409833+0000","last_clean":"2023-10-09T06:23:19.409833+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:23:19.409833+0000","last_undegraded":"2023-10-09T06:23:19.409833+0000","last_fullsized":"2023-10-09T06:23:19.409833+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"146'2589","reported_seq":2602,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:46.578610+0000","last_change":"2023-10-09T06:23:02.812031+0000","last_active":"2023-10-09T06:23:46.578610+0000","last_peered":"2023-10-09T06:23:46.578610+0000","last_clean":"2023-10-09T06:23:46.578610+0000","last_became_active":"2023-10-09T06:23:02.811742+0000","last_became_peered":"2023-10-09T06:23:02.811742+0000","last_unstale":"2023-10-09T06:23:46.578610+0000","last_undegraded":"2023-10-09T06:23:46.578610+0000","last_fullsized":"2023-10-09T06:23:46.578610+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2589,"log_dups_size":0,"ondisk_log_size":2589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.791030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169672704,"num_objects":2589,"num_object_clones":0,"num_object_copies":7767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2589,"num_write_kb":165696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410446+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:23:19.410446+0000","last_peered":"2023-10-09T06:23:19.410446+0000","last_clean":"2023-10-09T06:23:19.410446+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:23:19.410446+0000","last_undegraded":"2023-10-09T06:23:19.410446+0000","last_fullsized":"2023-10-09T06:23:19.410446+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410014+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:23:19.410014+0000","last_peered":"2023-10-09T06:23:19.410014+0000","last_clean":"2023-10-09T06:23:19.410014+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:23:19.410014+0000","last_undegraded":"2023-10-09T06:23:19.410014+0000","last_fullsized":"2023-10-09T06:23:19.410014+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"146'2553","reported_seq":2566,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:45.712036+0000","last_change":"2023-10-09T06:23:02.813423+0000","last_active":"2023-10-09T06:23:45.712036+0000","last_peered":"2023-10-09T06:23:45.712036+0000","last_clean":"2023-10-09T06:23:45.712036+0000","last_became_active":"2023-10-09T06:23:02.813075+0000","last_became_peered":"2023-10-09T06:23:02.813075+0000","last_unstale":"2023-10-09T06:23:45.712036+0000","last_undegraded":"2023-10-09T06:23:45.712036+0000","last_fullsized":"2023-10-09T06:23:45.712036+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2553,"log_dups_size":0,"ondisk_log_size":2553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:07.790690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2553,"num_write_kb":163392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":129,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.411148+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:23:19.411148+0000","last_peered":"2023-10-09T06:23:19.411148+0000","last_clean":"2023-10-09T06:23:19.411148+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:23:19.411148+0000","last_undegraded":"2023-10-09T06:23:19.411148+0000","last_fullsized":"2023-10-09T06:23:19.411148+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410853+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:23:19.410853+0000","last_peered":"2023-10-09T06:23:19.410853+0000","last_clean":"2023-10-09T06:23:19.410853+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:23:19.410853+0000","last_undegraded":"2023-10-09T06:23:19.410853+0000","last_fullsized":"2023-10-09T06:23:19.410853+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"146'2452","reported_seq":2464,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.419496+0000","last_change":"2023-10-09T06:23:02.819776+0000","last_active":"2023-10-09T06:23:44.419496+0000","last_peered":"2023-10-09T06:23:44.419496+0000","last_clean":"2023-10-09T06:23:44.419496+0000","last_became_active":"2023-10-09T06:23:02.819408+0000","last_became_peered":"2023-10-09T06:23:02.819408+0000","last_unstale":"2023-10-09T06:23:44.419496+0000","last_undegraded":"2023-10-09T06:23:44.419496+0000","last_fullsized":"2023-10-09T06:23:44.419496+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2452,"log_dups_size":0,"ondisk_log_size":2452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160694272,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2452,"num_write_kb":156928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.810218+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:23:20.810218+0000","last_peered":"2023-10-09T06:23:20.810218+0000","last_clean":"2023-10-09T06:23:20.810218+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:23:20.810218+0000","last_undegraded":"2023-10-09T06:23:20.810218+0000","last_fullsized":"2023-10-09T06:23:20.810218+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":94,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401347+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:23:19.401347+0000","last_peered":"2023-10-09T06:23:19.401347+0000","last_clean":"2023-10-09T06:23:19.401347+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:23:19.401347+0000","last_undegraded":"2023-10-09T06:23:19.401347+0000","last_fullsized":"2023-10-09T06:23:19.401347+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"146'2841","reported_seq":2854,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:48.176607+0000","last_change":"2023-10-09T06:23:02.808018+0000","last_active":"2023-10-09T06:23:48.176607+0000","last_peered":"2023-10-09T06:23:48.176607+0000","last_clean":"2023-10-09T06:23:48.176607+0000","last_became_active":"2023-10-09T06:23:02.807707+0000","last_became_peered":"2023-10-09T06:23:02.807707+0000","last_unstale":"2023-10-09T06:23:48.176607+0000","last_undegraded":"2023-10-09T06:23:48.176607+0000","last_fullsized":"2023-10-09T06:23:48.176607+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2841,"log_dups_size":0,"ondisk_log_size":2841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.598902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2841,"num_write_kb":181824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":994,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400899+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:23:19.400899+0000","last_peered":"2023-10-09T06:23:19.400899+0000","last_clean":"2023-10-09T06:23:19.400899+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:23:19.400899+0000","last_undegraded":"2023-10-09T06:23:19.400899+0000","last_fullsized":"2023-10-09T06:23:19.400899+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900723+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:23:18.900723+0000","last_peered":"2023-10-09T06:23:18.900723+0000","last_clean":"2023-10-09T06:23:18.900723+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:23:18.900723+0000","last_undegraded":"2023-10-09T06:23:18.900723+0000","last_fullsized":"2023-10-09T06:23:18.900723+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400887+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:23:19.400887+0000","last_peered":"2023-10-09T06:23:19.400887+0000","last_clean":"2023-10-09T06:23:19.400887+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:23:19.400887+0000","last_undegraded":"2023-10-09T06:23:19.400887+0000","last_fullsized":"2023-10-09T06:23:19.400887+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"146'2587","reported_seq":2600,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:45.713059+0000","last_change":"2023-10-09T06:23:02.809922+0000","last_active":"2023-10-09T06:23:45.713059+0000","last_peered":"2023-10-09T06:23:45.713059+0000","last_clean":"2023-10-09T06:23:45.713059+0000","last_became_active":"2023-10-09T06:23:02.809657+0000","last_became_peered":"2023-10-09T06:23:02.809657+0000","last_unstale":"2023-10-09T06:23:45.713059+0000","last_undegraded":"2023-10-09T06:23:45.713059+0000","last_fullsized":"2023-10-09T06:23:45.713059+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2587,"log_dups_size":0,"ondisk_log_size":2587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:22.193166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2587,"num_write_kb":165568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":727,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900197+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:23:18.900197+0000","last_peered":"2023-10-09T06:23:18.900197+0000","last_clean":"2023-10-09T06:23:18.900197+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:23:18.900197+0000","last_undegraded":"2023-10-09T06:23:18.900197+0000","last_fullsized":"2023-10-09T06:23:18.900197+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410411+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:23:19.410411+0000","last_peered":"2023-10-09T06:23:19.410411+0000","last_clean":"2023-10-09T06:23:19.410411+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:23:19.410411+0000","last_undegraded":"2023-10-09T06:23:19.410411+0000","last_fullsized":"2023-10-09T06:23:19.410411+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400750+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:23:19.400750+0000","last_peered":"2023-10-09T06:23:19.400750+0000","last_clean":"2023-10-09T06:23:19.400750+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:23:19.400750+0000","last_undegraded":"2023-10-09T06:23:19.400750+0000","last_fullsized":"2023-10-09T06:23:19.400750+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"146'2427","reported_seq":2439,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.408341+0000","last_change":"2023-10-09T06:23:02.818151+0000","last_active":"2023-10-09T06:23:44.408341+0000","last_peered":"2023-10-09T06:23:44.408341+0000","last_clean":"2023-10-09T06:23:44.408341+0000","last_became_active":"2023-10-09T06:23:02.816332+0000","last_became_peered":"2023-10-09T06:23:02.816332+0000","last_unstale":"2023-10-09T06:23:44.408341+0000","last_undegraded":"2023-10-09T06:23:44.408341+0000","last_fullsized":"2023-10-09T06:23:44.408341+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2427,"log_dups_size":0,"ondisk_log_size":2427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055872,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2427,"num_write_kb":155328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":505,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.876514+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:23:18.876514+0000","last_peered":"2023-10-09T06:23:18.876514+0000","last_clean":"2023-10-09T06:23:18.876514+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:23:18.876514+0000","last_undegraded":"2023-10-09T06:23:18.876514+0000","last_fullsized":"2023-10-09T06:23:18.876514+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":94,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.371559+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:23:21.371559+0000","last_peered":"2023-10-09T06:23:21.371559+0000","last_clean":"2023-10-09T06:23:21.371559+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:23:21.371559+0000","last_undegraded":"2023-10-09T06:23:21.371559+0000","last_fullsized":"2023-10-09T06:23:21.371559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":123,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401371+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:23:19.401371+0000","last_peered":"2023-10-09T06:23:19.401371+0000","last_clean":"2023-10-09T06:23:19.401371+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:23:19.401371+0000","last_undegraded":"2023-10-09T06:23:19.401371+0000","last_fullsized":"2023-10-09T06:23:19.401371+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"146'2485","reported_seq":2497,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.417414+0000","last_change":"2023-10-09T06:23:02.821017+0000","last_active":"2023-10-09T06:23:44.417414+0000","last_peered":"2023-10-09T06:23:44.417414+0000","last_clean":"2023-10-09T06:23:44.417414+0000","last_became_active":"2023-10-09T06:23:02.820708+0000","last_became_peered":"2023-10-09T06:23:02.820708+0000","last_unstale":"2023-10-09T06:23:44.417414+0000","last_undegraded":"2023-10-09T06:23:44.417414+0000","last_fullsized":"2023-10-09T06:23:44.417414+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2485,"log_dups_size":0,"ondisk_log_size":2485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":159040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":494,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900292+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:23:18.900292+0000","last_peered":"2023-10-09T06:23:18.900292+0000","last_clean":"2023-10-09T06:23:18.900292+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:23:18.900292+0000","last_undegraded":"2023-10-09T06:23:18.900292+0000","last_fullsized":"2023-10-09T06:23:18.900292+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":342,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.327616+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:23:19.327616+0000","last_peered":"2023-10-09T06:23:19.327616+0000","last_clean":"2023-10-09T06:23:19.327616+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:23:19.327616+0000","last_undegraded":"2023-10-09T06:23:19.327616+0000","last_fullsized":"2023-10-09T06:23:19.327616+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878950+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:23:18.878950+0000","last_peered":"2023-10-09T06:23:18.878950+0000","last_clean":"2023-10-09T06:23:18.878950+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:23:18.878950+0000","last_undegraded":"2023-10-09T06:23:18.878950+0000","last_fullsized":"2023-10-09T06:23:18.878950+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410874+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:23:19.410874+0000","last_peered":"2023-10-09T06:23:19.410874+0000","last_clean":"2023-10-09T06:23:19.410874+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:23:19.410874+0000","last_undegraded":"2023-10-09T06:23:19.410874+0000","last_fullsized":"2023-10-09T06:23:19.410874+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"146'2744","reported_seq":2757,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:46.686688+0000","last_change":"2023-10-09T06:23:02.811831+0000","last_active":"2023-10-09T06:23:46.686688+0000","last_peered":"2023-10-09T06:23:46.686688+0000","last_clean":"2023-10-09T06:23:46.686688+0000","last_became_active":"2023-10-09T06:23:02.811445+0000","last_became_peered":"2023-10-09T06:23:02.811445+0000","last_unstale":"2023-10-09T06:23:46.686688+0000","last_undegraded":"2023-10-09T06:23:46.686688+0000","last_fullsized":"2023-10-09T06:23:46.686688+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2744,"log_dups_size":0,"ondisk_log_size":2744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:28.021647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2744,"num_write_kb":175616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":67,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.327405+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:23:19.327405+0000","last_peered":"2023-10-09T06:23:19.327405+0000","last_clean":"2023-10-09T06:23:19.327405+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:23:19.327405+0000","last_undegraded":"2023-10-09T06:23:19.327405+0000","last_fullsized":"2023-10-09T06:23:19.327405+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":98,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.942912+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:23:20.942912+0000","last_peered":"2023-10-09T06:23:20.942912+0000","last_clean":"2023-10-09T06:23:20.942912+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:23:20.942912+0000","last_undegraded":"2023-10-09T06:23:20.942912+0000","last_fullsized":"2023-10-09T06:23:20.942912+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":112,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.402219+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:23:19.402219+0000","last_peered":"2023-10-09T06:23:19.402219+0000","last_clean":"2023-10-09T06:23:19.402219+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:23:19.402219+0000","last_undegraded":"2023-10-09T06:23:19.402219+0000","last_fullsized":"2023-10-09T06:23:19.402219+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"146'2667","reported_seq":2680,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:46.708863+0000","last_change":"2023-10-09T06:23:02.810207+0000","last_active":"2023-10-09T06:23:46.708863+0000","last_peered":"2023-10-09T06:23:46.708863+0000","last_clean":"2023-10-09T06:23:46.708863+0000","last_became_active":"2023-10-09T06:23:02.809844+0000","last_became_peered":"2023-10-09T06:23:02.809844+0000","last_unstale":"2023-10-09T06:23:46.708863+0000","last_undegraded":"2023-10-09T06:23:46.708863+0000","last_fullsized":"2023-10-09T06:23:46.708863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2667,"log_dups_size":0,"ondisk_log_size":2667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:38.667224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2667,"num_write_kb":170688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":56,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.328571+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:23:19.328571+0000","last_peered":"2023-10-09T06:23:19.328571+0000","last_clean":"2023-10-09T06:23:19.328571+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:23:19.328571+0000","last_undegraded":"2023-10-09T06:23:19.328571+0000","last_fullsized":"2023-10-09T06:23:19.328571+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":93,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.400793+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:23:19.400793+0000","last_peered":"2023-10-09T06:23:19.400793+0000","last_clean":"2023-10-09T06:23:19.400793+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:23:19.400793+0000","last_undegraded":"2023-10-09T06:23:19.400793+0000","last_fullsized":"2023-10-09T06:23:19.400793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943227+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:23:20.943227+0000","last_peered":"2023-10-09T06:23:20.943227+0000","last_clean":"2023-10-09T06:23:20.943227+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:23:20.943227+0000","last_undegraded":"2023-10-09T06:23:20.943227+0000","last_fullsized":"2023-10-09T06:23:20.943227+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"146'2532","reported_seq":2544,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.411147+0000","last_change":"2023-10-09T06:23:02.821182+0000","last_active":"2023-10-09T06:23:44.411147+0000","last_peered":"2023-10-09T06:23:44.411147+0000","last_clean":"2023-10-09T06:23:44.411147+0000","last_became_active":"2023-10-09T06:23:02.820839+0000","last_became_peered":"2023-10-09T06:23:02.820839+0000","last_unstale":"2023-10-09T06:23:44.411147+0000","last_undegraded":"2023-10-09T06:23:44.411147+0000","last_fullsized":"2023-10-09T06:23:44.411147+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2532,"log_dups_size":0,"ondisk_log_size":2532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2532,"num_write_kb":162048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":143,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.810563+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:23:20.810563+0000","last_peered":"2023-10-09T06:23:20.810563+0000","last_clean":"2023-10-09T06:23:20.810563+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:23:20.810563+0000","last_undegraded":"2023-10-09T06:23:20.810563+0000","last_fullsized":"2023-10-09T06:23:20.810563+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":450,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.401243+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:23:19.401243+0000","last_peered":"2023-10-09T06:23:19.401243+0000","last_clean":"2023-10-09T06:23:19.401243+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:23:19.401243+0000","last_undegraded":"2023-10-09T06:23:19.401243+0000","last_fullsized":"2023-10-09T06:23:19.401243+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":141,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878699+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:23:18.878699+0000","last_peered":"2023-10-09T06:23:18.878699+0000","last_clean":"2023-10-09T06:23:18.878699+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:23:18.878699+0000","last_undegraded":"2023-10-09T06:23:18.878699+0000","last_fullsized":"2023-10-09T06:23:18.878699+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"146'2566","reported_seq":2578,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:44.426729+0000","last_change":"2023-10-09T06:23:02.819624+0000","last_active":"2023-10-09T06:23:44.426729+0000","last_peered":"2023-10-09T06:23:44.426729+0000","last_clean":"2023-10-09T06:23:44.426729+0000","last_became_active":"2023-10-09T06:23:02.818630+0000","last_became_peered":"2023-10-09T06:23:02.818630+0000","last_unstale":"2023-10-09T06:23:44.426729+0000","last_undegraded":"2023-10-09T06:23:44.426729+0000","last_fullsized":"2023-10-09T06:23:44.426729+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":2566,"log_dups_size":0,"ondisk_log_size":2566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168165376,"num_objects":2566,"num_object_clones":0,"num_object_copies":7698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2566,"num_write_kb":164224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":842,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409190+0000","last_change":"2023-10-09T06:22:46.890972+0000","last_active":"2023-10-09T06:23:19.409190+0000","last_peered":"2023-10-09T06:23:19.409190+0000","last_clean":"2023-10-09T06:23:19.409190+0000","last_became_active":"2023-10-09T06:22:46.889916+0000","last_became_peered":"2023-10-09T06:22:46.889916+0000","last_unstale":"2023-10-09T06:23:19.409190+0000","last_undegraded":"2023-10-09T06:23:19.409190+0000","last_fullsized":"2023-10-09T06:23:19.409190+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":101,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.402064+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:23:19.402064+0000","last_peered":"2023-10-09T06:23:19.402064+0000","last_clean":"2023-10-09T06:23:19.402064+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:23:19.402064+0000","last_undegraded":"2023-10-09T06:23:19.402064+0000","last_fullsized":"2023-10-09T06:23:19.402064+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409819+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:23:19.409819+0000","last_peered":"2023-10-09T06:23:19.409819+0000","last_clean":"2023-10-09T06:23:19.409819+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:23:19.409819+0000","last_undegraded":"2023-10-09T06:23:19.409819+0000","last_fullsized":"2023-10-09T06:23:19.409819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":104,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.402310+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:23:19.402310+0000","last_peered":"2023-10-09T06:23:19.402310+0000","last_clean":"2023-10-09T06:23:19.402310+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:23:19.402310+0000","last_undegraded":"2023-10-09T06:23:19.402310+0000","last_fullsized":"2023-10-09T06:23:19.402310+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.876341+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:23:18.876341+0000","last_peered":"2023-10-09T06:23:18.876341+0000","last_clean":"2023-10-09T06:23:18.876341+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:23:18.876341+0000","last_undegraded":"2023-10-09T06:23:18.876341+0000","last_fullsized":"2023-10-09T06:23:18.876341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":113,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900811+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:23:18.900811+0000","last_peered":"2023-10-09T06:23:18.900811+0000","last_clean":"2023-10-09T06:23:18.900811+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:23:18.900811+0000","last_undegraded":"2023-10-09T06:23:18.900811+0000","last_fullsized":"2023-10-09T06:23:18.900811+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410716+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:23:19.410716+0000","last_peered":"2023-10-09T06:23:19.410716+0000","last_clean":"2023-10-09T06:23:19.410716+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:23:19.410716+0000","last_undegraded":"2023-10-09T06:23:19.410716+0000","last_fullsized":"2023-10-09T06:23:19.410716+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":98,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.900755+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:23:18.900755+0000","last_peered":"2023-10-09T06:23:18.900755+0000","last_clean":"2023-10-09T06:23:18.900755+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:23:18.900755+0000","last_undegraded":"2023-10-09T06:23:18.900755+0000","last_fullsized":"2023-10-09T06:23:18.900755+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.410341+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:23:19.410341+0000","last_peered":"2023-10-09T06:23:19.410341+0000","last_clean":"2023-10-09T06:23:19.410341+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:23:19.410341+0000","last_undegraded":"2023-10-09T06:23:19.410341+0000","last_fullsized":"2023-10-09T06:23:19.410341+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":124,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.879819+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:23:18.879819+0000","last_peered":"2023-10-09T06:23:18.879819+0000","last_clean":"2023-10-09T06:23:18.879819+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:23:18.879819+0000","last_undegraded":"2023-10-09T06:23:18.879819+0000","last_fullsized":"2023-10-09T06:23:18.879819+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:18.878529+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:23:18.878529+0000","last_peered":"2023-10-09T06:23:18.878529+0000","last_clean":"2023-10-09T06:23:18.878529+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:23:18.878529+0000","last_undegraded":"2023-10-09T06:23:18.878529+0000","last_fullsized":"2023-10-09T06:23:18.878529+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":102,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943445+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:23:20.943445+0000","last_peered":"2023-10-09T06:23:20.943445+0000","last_clean":"2023-10-09T06:23:20.943445+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:23:20.943445+0000","last_undegraded":"2023-10-09T06:23:20.943445+0000","last_fullsized":"2023-10-09T06:23:20.943445+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-10-09T06:23:19.409466+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:23:19.409466+0000","last_peered":"2023-10-09T06:23:19.409466+0000","last_clean":"2023-10-09T06:23:19.409466+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:23:19.409466+0000","last_undegraded":"2023-10-09T06:23:19.409466+0000","last_fullsized":"2023-10-09T06:23:19.409466+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2712797184,"num_objects":41394,"num_object_clones":0,"num_object_copies":124182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41394,"num_write_kb":2649216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8153464832,"data_stored":8153464832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41394,"ondisk_log_size":41394,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430509,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1256452,"kb_used_data":955132,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92488700,"statfs":{"total":95995035648,"available":94708428800,"internally_reserved":0,"allocated":978055168,"data_stored":977529233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.4319999999999999,"15min":1.3979999999999999},"min":{"1min":0.44800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.5129999999999999,"5min":107.36799999999999,"15min":107.36799999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.4209999999999998,"15min":1.387},"min":{"1min":0.49399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.131,"5min":105.89,"15min":105.89},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:23:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":2.883,"15min":1.55},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.7199999999999998,"5min":104.825,"15min":104.825},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.296,"5min":2.46,"15min":1.4430000000000001},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.6799999999999997,"5min":104.637,"15min":104.637},"last":0.877}]},{"osd":2,"last update":"Mon Oct 9 06:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":4.1470000000000002,"15min":1.998},"min":{"1min":0.52800000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.093,"5min":104.992,"15min":104.992},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.0009999999999999,"5min":4.0369999999999999,"15min":2.0129999999999999},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.3150000000000004,"5min":104.691,"15min":104.691},"last":0.68500000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":1.4490000000000001,"15min":1.1559999999999999},"min":{"1min":0.48899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.46,"5min":9.6869999999999994,"15min":9.6869999999999994},"last":137.11600000000001},{"interface":"front","average":{"1min":1.155,"5min":1.163,"15min":1.151},"min":{"1min":0.40999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":4.3609999999999998,"5min":9.1969999999999992,"15min":9.1969999999999992},"last":137.14400000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.7909999999999999,"15min":1.204},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29399999999999998},"max":{"1min":1.2470000000000001,"5min":11.029999999999999,"15min":11.029999999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.87,"5min":1.948,"15min":1.276},"min":{"1min":0.52900000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.296,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":0.83299999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.8280000000000001,"15min":1.3120000000000001},"min":{"1min":0.41499999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.4009999999999998,"5min":11.996,"15min":11.996},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.752,"15min":1.2949999999999999},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.327,"5min":13.634,"15min":13.634},"last":0.96599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.3939999999999999,"15min":1.1839999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3859999999999999,"5min":9.9510000000000005,"15min":9.9510000000000005},"last":1.036},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.669,"15min":1.2809999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3400000000000001,"5min":8.6199999999999992,"15min":8.6199999999999992},"last":0.995}]}]},{"osd":6,"up_from":42,"seq":180388626741,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1083572,"kb_used_data":786348,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92661580,"statfs":{"total":95995035648,"available":94885457920,"internally_reserved":0,"allocated":805220352,"data_stored":804710044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.7,"15min":1.161},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.1309999999999998,"5min":32.289999999999999,"15min":32.289999999999999},"last":3.4260000000000002},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.573,"15min":1.1379999999999999},"min":{"1min":0.45600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.6699999999999999,"5min":31.454999999999998,"15min":31.454999999999998},"last":0.57099999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.6219999999999999,"15min":1.454},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9249999999999998,"5min":137.93899999999999,"15min":137.93899999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.1379999999999999,"5min":2.673,"15min":1.4930000000000001},"min":{"1min":0.35199999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.1529999999999996,"5min":138.02500000000001,"15min":138.02500000000001},"last":3.472}]},{"osd":2,"last update":"Mon Oct 9 06:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.98,"15min":1.639},"min":{"1min":0.63100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.4749999999999996,"5min":95.540000000000006,"15min":95.540000000000006},"last":2.5249999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.9580000000000002,"15min":1.6499999999999999},"min":{"1min":0.5,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":96.352000000000004,"15min":96.352000000000004},"last":0.627}]},{"osd":3,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.3109999999999999,"15min":1.137},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.2290000000000001,"5min":7.3410000000000002,"15min":7.3410000000000002},"last":5.2859999999999996},{"interface":"front","average":{"1min":1.542,"5min":1.1259999999999999,"15min":1.046},"min":{"1min":0.38,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.4860000000000007,"5min":9.4860000000000007,"15min":9.4860000000000007},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.6060000000000001,"15min":1.163},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.3310000000000004,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":2.9300000000000002},{"interface":"front","average":{"1min":1.403,"5min":1.746,"15min":1.2470000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.851,"5min":10.300000000000001,"15min":10.300000000000001},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:23:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.8220000000000001,"15min":1.2889999999999999},"min":{"1min":0.46000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.766,"5min":11.754,"15min":11.754},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.8049999999999999,"15min":1.3029999999999999},"min":{"1min":0.38800000000000001,"5min":0.308,"15min":0.308},"max":{"1min":9.0419999999999998,"5min":12.295999999999999,"15min":12.295999999999999},"last":1.4139999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:22:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.7709999999999999,"15min":1.3200000000000001},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.4179999999999993,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.5070000000000001,"5min":1.921,"15min":1.3959999999999999},"min":{"1min":0.438,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.102,"5min":11.102,"15min":11.102},"last":5.3479999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958651,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":965592,"kb_used_data":839568,"kb_used_omap":7,"kb_used_meta":126008,"kb_avail":92779560,"statfs":{"total":95995035648,"available":95006269440,"internally_reserved":0,"allocated":859717632,"data_stored":859059878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":129032592},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.4350000000000001,"15min":0.94999999999999996},"min":{"1min":0.45000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.698,"5min":31.969000000000001,"15min":31.969000000000001},"last":3.9550000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.8839999999999999,"15min":1.085},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.8879999999999999,"5min":40.667999999999999,"15min":40.667999999999999},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":1.2410000000000001,"15min":0.90200000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.4220000000000002,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.278,"15min":0.999},"min":{"1min":0.40600000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.208,"5min":8.3960000000000008,"15min":8.3960000000000008},"last":1.6899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.2909999999999999,"15min":0.98899999999999999},"min":{"1min":0.34999999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.6539999999999999,"5min":16.390000000000001,"15min":16.390000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.1220000000000001,"15min":0.88400000000000001},"min":{"1min":0.34100000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.657,"5min":16.555,"15min":16.555},"last":4.0529999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.5069999999999999,"15min":1.1619999999999999},"min":{"1min":0.48699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.7469999999999999,"5min":6.7469999999999999,"15min":6.7469999999999999},"last":1.806},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.6519999999999999,"15min":1.2110000000000001},"min":{"1min":0.441,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.6340000000000003,"5min":7.3639999999999999,"15min":7.3639999999999999},"last":3.9159999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.748,"15min":1.2609999999999999},"min":{"1min":0.49299999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.871,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.843,"5min":1.8260000000000001,"15min":1.3120000000000001},"min":{"1min":0.50700000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.7960000000000003,"5min":6.7960000000000003,"15min":6.7960000000000003},"last":1.917}]},{"osd":6,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.4350000000000001,"15min":1.1859999999999999},"min":{"1min":0.39100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.1189999999999998,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.512,"15min":1.258},"min":{"1min":0.45400000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.7720000000000002,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":0.68700000000000006}]},{"osd":7,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.925,"5min":2.0249999999999999,"15min":1.4279999999999999},"min":{"1min":0.76700000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":6.7229999999999999,"5min":7.46,"15min":7.46},"last":1.7290000000000001},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.653,"15min":1.3160000000000001},"min":{"1min":0.48599999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":6.9180000000000001,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":4.0270000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122177,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1261476,"kb_used_data":1085340,"kb_used_omap":8,"kb_used_meta":176055,"kb_avail":92483676,"statfs":{"total":95995035648,"available":94703284224,"internally_reserved":0,"allocated":1111388160,"data_stored":1110718979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":180280576},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.6890000000000001,"15min":0.97899999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.1750000000000007,"5min":12.460000000000001,"15min":12.460000000000001},"last":4.3209999999999997},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.5149999999999999,"15min":0.99199999999999999},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.165,"5min":10.243,"15min":10.243},"last":4.4790000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:23:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.5109999999999999,"15min":0.95099999999999996},"min":{"1min":0.28399999999999997,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.0860000000000001,"5min":13.035,"15min":13.035},"last":4.3659999999999997},{"interface":"front","average":{"1min":0.622,"5min":1.7030000000000001,"15min":1.0569999999999999},"min":{"1min":0.33900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.95999999999999996,"5min":14.896000000000001,"15min":14.896000000000001},"last":4.609}]},{"osd":3,"last update":"Mon Oct 9 06:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.288,"15min":0.92800000000000005},"min":{"1min":0.255,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.7059999999999995,"5min":8.7059999999999995,"15min":8.7059999999999995},"last":5.3230000000000004},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.0700000000000001,"15min":0.86799999999999999},"min":{"1min":0.35699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.2549999999999999,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":4.577}]},{"osd":4,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.762,"15min":1.2370000000000001},"min":{"1min":0.443,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.3630000000000004,"5min":9.0890000000000004,"15min":9.0890000000000004},"last":5.3630000000000004},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.889,"15min":1.3169999999999999},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":6.0650000000000004,"5min":10.505000000000001,"15min":10.505000000000001},"last":6.0650000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.1259999999999999,"15min":1.415},"min":{"1min":0.45500000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.3390000000000004,"5min":9.548,"15min":9.548},"last":5.3390000000000004},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.8140000000000001,"15min":1.3109999999999999},"min":{"1min":0.58199999999999996,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.6360000000000001,"5min":10.329000000000001,"15min":10.329000000000001},"last":4.6360000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.548,"15min":1.2250000000000001},"min":{"1min":0.47699999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6589999999999998,"5min":10.194000000000001,"15min":10.194000000000001},"last":4.6589999999999998},{"interface":"front","average":{"1min":1.097,"5min":1.512,"15min":1.1970000000000001},"min":{"1min":0.496,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.4580000000000002,"5min":8.7629999999999999,"15min":8.7629999999999999},"last":5.4580000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.7190000000000001,"15min":1.333},"min":{"1min":0.60199999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.7480000000000002,"5min":6.0330000000000004,"15min":6.0330000000000004},"last":4.7480000000000002},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.9850000000000001,"15min":1.4179999999999999},"min":{"1min":0.51200000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":5.3849999999999998,"5min":8.7959999999999994,"15min":8.7959999999999994},"last":5.3849999999999998}]}]},{"osd":2,"up_from":98,"seq":420906795125,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503760,"kb_used_data":1341704,"kb_used_omap":11,"kb_used_meta":162036,"kb_avail":92241392,"statfs":{"total":95995035648,"available":94455185408,"internally_reserved":0,"allocated":1373904896,"data_stored":1373250922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":165925822},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":2.1880000000000002,"15min":1.198},"min":{"1min":0.29599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":8.609,"5min":35.015999999999998,"15min":35.015999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.1399999999999999,"5min":2.411,"15min":1.262},"min":{"1min":0.32800000000000001,"5min":0.182,"15min":0.182},"max":{"1min":6.7469999999999999,"5min":36.415999999999997,"15min":36.415999999999997},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.395,"15min":0.95299999999999996},"min":{"1min":0.40699999999999997,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.5399999999999991,"5min":8.5399999999999991,"15min":8.5399999999999991},"last":3.5659999999999998},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.9650000000000001,"15min":1.143},"min":{"1min":0.34999999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.0510000000000002,"5min":10.4,"15min":10.4},"last":0.29299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.371,"15min":0.94899999999999995},"min":{"1min":0.35299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.311999999999999,"5min":13.311999999999999,"15min":13.311999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.2190000000000001,"15min":0.875},"min":{"1min":0.35599999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.306,"5min":18.831,"15min":18.831},"last":0.63200000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":2.5760000000000001,"15min":1.51},"min":{"1min":0.57199999999999995,"5min":0.315,"15min":0.315},"max":{"1min":3.1179999999999999,"5min":19.184000000000001,"15min":19.184000000000001},"last":2.1259999999999999},{"interface":"front","average":{"1min":1.2090000000000001,"5min":2.2869999999999999,"15min":1.3819999999999999},"min":{"1min":0.63300000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.2130000000000001,"5min":29.408000000000001,"15min":29.408000000000001},"last":1.054}]},{"osd":5,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.2330000000000001,"15min":1.4239999999999999},"min":{"1min":0.53000000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.0819999999999999,"5min":12.833,"15min":12.833},"last":2.194},{"interface":"front","average":{"1min":1.159,"5min":2.0350000000000001,"15min":1.3089999999999999},"min":{"1min":0.62,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.96,"5min":29.545000000000002,"15min":29.545000000000002},"last":0.84999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.605,"15min":1.2010000000000001},"min":{"1min":0.55100000000000005,"5min":0.372,"15min":0.372},"max":{"1min":3.8860000000000001,"5min":29.504000000000001,"15min":29.504000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.637,"5min":1.7549999999999999,"15min":1.2769999999999999},"min":{"1min":0.53600000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":7.0970000000000004,"5min":8.1349999999999998,"15min":8.1349999999999998},"last":3.4249999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.8879999999999999,"15min":1.377},"min":{"1min":0.46700000000000003,"5min":0.308,"15min":0.308},"max":{"1min":7.165,"5min":8.3309999999999995,"15min":8.3309999999999995},"last":2.1659999999999999},{"interface":"front","average":{"1min":1.1859999999999999,"5min":2.3010000000000002,"15min":1.5},"min":{"1min":0.56000000000000005,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.2229999999999999,"5min":10.978999999999999,"15min":10.978999999999999},"last":2.2549999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631599,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1734984,"kb_used_data":1635392,"kb_used_omap":3,"kb_used_meta":99580,"kb_avail":92010168,"statfs":{"total":95995035648,"available":94218412032,"internally_reserved":0,"allocated":1674641408,"data_stored":1673994854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":101970465},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.978,"15min":1.105},"min":{"1min":0.215,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":1.6399999999999999,"5min":62.939,"15min":62.939},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.1309999999999998,"15min":1.1459999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":4.7590000000000003,"5min":63.869,"15min":63.869},"last":0.27900000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.0089999999999999,"15min":0.80900000000000005},"min":{"1min":0.27900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.8570000000000002,"5min":5.944,"15min":5.944},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.0580000000000001,"15min":0.86399999999999999},"min":{"1min":0.40500000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.4329999999999998,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":1.1579999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.2490000000000001,"15min":1.294},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.78,"5min":70.594999999999999,"15min":70.594999999999999},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.002,"5min":1.859,"15min":1.113},"min":{"1min":0.29199999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.8339999999999996,"5min":62.991999999999997,"15min":62.991999999999997},"last":0.63300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.804,"15min":1.2170000000000001},"min":{"1min":0.42199999999999999,"5min":0.314,"15min":0.314},"max":{"1min":4.7130000000000001,"5min":10.446999999999999,"15min":10.446999999999999},"last":1.8779999999999999},{"interface":"front","average":{"1min":1.613,"5min":1.667,"15min":1.218},"min":{"1min":0.43099999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.5830000000000002,"5min":7.8940000000000001,"15min":7.8940000000000001},"last":1.925}]},{"osd":5,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.446,"15min":1.171},"min":{"1min":0.46500000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.0679999999999996,"5min":6.0910000000000002,"15min":6.0910000000000002},"last":2.7770000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.7669999999999999,"15min":1.3029999999999999},"min":{"1min":0.40400000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.7370000000000001,"5min":11.754,"15min":11.754},"last":1.218}]},{"osd":6,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.232,"5min":1.1499999999999999,"15min":1.127},"min":{"1min":0.39000000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.6360000000000001,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":2.2370000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.3660000000000001,"15min":1.1839999999999999},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.1150000000000002,"5min":6.0250000000000004,"15min":6.0250000000000004},"last":0.72199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.4790000000000001,"15min":1.202},"min":{"1min":0.55000000000000004,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.6669999999999998,"5min":6.4820000000000002,"15min":6.4820000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.359,"5min":1.512,"15min":1.262},"min":{"1min":0.46000000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.681,"5min":11.439,"15min":11.439},"last":2.2869999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019207,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1282236,"kb_used_data":972724,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92462916,"statfs":{"total":95995035648,"available":94682025984,"internally_reserved":0,"allocated":996069376,"data_stored":995551019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.2029999999999998,"15min":1.861},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5780000000000003,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.5150000000000001,"15min":1.9930000000000001},"min":{"1min":0.41299999999999998,"5min":0.312,"15min":0.312},"max":{"1min":4.6310000000000002,"5min":84.912999999999997,"15min":84.912999999999997},"last":2.7679999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":2.3559999999999999,"15min":1.4319999999999999},"min":{"1min":0.38100000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.919,"5min":61.726999999999997,"15min":61.726999999999997},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.5449999999999999,"5min":3.0539999999999998,"15min":1.677},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.265000000000001,"5min":61.932000000000002,"15min":61.932000000000002},"last":1.0029999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":3.2719999999999998,"15min":1.718},"min":{"1min":0.35499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.103,"5min":61.783999999999999,"15min":61.783999999999999},"last":2.8399999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.4860000000000002,"15min":1.784},"min":{"1min":0.53100000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.2909999999999999,"5min":61.551000000000002,"15min":61.551000000000002},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.29,"15min":1.448},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.0979999999999999,"5min":36.154000000000003,"15min":36.154000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.6890000000000001,"15min":1.254},"min":{"1min":0.40600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":9.3490000000000002,"5min":36.064,"15min":36.064},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:23:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.8109999999999999,"15min":1.2230000000000001},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.0030000000000001,"5min":12.242000000000001,"15min":12.242000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.77100000000000002,"5min":2.5129999999999999,"15min":1.4930000000000001},"min":{"1min":0.34100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.2470000000000001,"5min":20.113,"15min":20.113},"last":2.875}]},{"osd":6,"last update":"Mon Oct 9 06:22:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":2.1749999999999998,"15min":1.413},"min":{"1min":0.374,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.8529999999999998,"5min":18.010000000000002,"15min":18.010000000000002},"last":2.7240000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.677,"15min":1.2410000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.867,"5min":13.913,"15min":13.913},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.306,"15min":1.496},"min":{"1min":0.58399999999999996,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.452,"5min":15.294,"15min":15.294},"last":2.8029999999999999},{"interface":"front","average":{"1min":1.012,"5min":2.3380000000000001,"15min":1.504},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.575,"5min":10.731999999999999,"15min":10.731999999999999},"last":1.575}]}]},{"osd":5,"up_from":36,"seq":154618822974,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110632,"kb_used_data":797024,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92634520,"statfs":{"total":95995035648,"available":94857748480,"internally_reserved":0,"allocated":816152576,"data_stored":815649832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":3.3849999999999998,"15min":1.804},"min":{"1min":0.311,"5min":0.247,"15min":0.247},"max":{"1min":1.1020000000000001,"5min":69.793999999999997,"15min":69.793999999999997},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.85999999999999999,"5min":2.847,"15min":1.5960000000000001},"min":{"1min":0.375,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2589999999999999,"5min":68.873000000000005,"15min":68.873000000000005},"last":0.39600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.5619999999999998,"15min":1.476},"min":{"1min":0.372,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3409999999999993,"5min":52.070999999999998,"15min":52.070999999999998},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.363,"5min":2.8570000000000002,"15min":1.5580000000000001},"min":{"1min":0.48999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.8700000000000001,"5min":51.570999999999998,"15min":51.570999999999998},"last":3.306}]},{"osd":2,"last update":"Mon Oct 9 06:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":3.109,"15min":1.6519999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.307},"max":{"1min":1.1859999999999999,"5min":47.207000000000001,"15min":47.207000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.875,"5min":3.0550000000000002,"15min":1.718},"min":{"1min":0.46600000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.347,"5min":59.082999999999998,"15min":59.082999999999998},"last":0.376}]},{"osd":3,"last update":"Mon Oct 9 06:23:00 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.851,"15min":1.292},"min":{"1min":0.40200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2439999999999998,"5min":12.334,"15min":12.334},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.361,"5min":1.6259999999999999,"15min":1.2390000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.4119999999999999,"5min":11.823,"15min":11.823},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.6709999999999998,"15min":1.494},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.669,"5min":17.667999999999999,"15min":17.667999999999999},"last":3.0539999999999998},{"interface":"front","average":{"1min":1.468,"5min":2.202,"15min":1.4119999999999999},"min":{"1min":0.52400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.9720000000000004,"5min":16.181999999999999,"15min":16.181999999999999},"last":0.75600000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.1110000000000002,"15min":1.361},"min":{"1min":0.495,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.9139999999999997,"5min":17.454999999999998,"15min":17.454999999999998},"last":3.1709999999999998},{"interface":"front","average":{"1min":1.677,"5min":2.0840000000000001,"15min":1.4330000000000001},"min":{"1min":0.58499999999999996,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":9.2919999999999998,"5min":12.365,"15min":12.365},"last":0.78500000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.339,"15min":1.532},"min":{"1min":0.64300000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.5740000000000001,"5min":13.047000000000001,"15min":13.047000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.1360000000000001,"15min":1.48},"min":{"1min":0.438,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.679,"5min":11.981,"15min":11.981},"last":0.41499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":734920704,"data_stored":734920704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":880148480,"data_stored":880148480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1400045568,"data_stored":1400045568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1702756352,"data_stored":1702756352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":815398912,"data_stored":815398912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":815661056,"data_stored":815661056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":821493760,"data_stored":821493760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":983040000,"data_stored":983040000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:23:53.498 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:23:53.499 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:23:53.499 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:23:53.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:23:53.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:53.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:53.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:53.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:53.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:53.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:53.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:53.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:53.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:54.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:54.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:54.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 52402 52386 109.122 79.625 0.0170388 0.0091582 2023-10-09T06:23:54.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:54.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:54.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:53 smithi125 bash[77299]: cluster 2023-10-09T06:23:52.894736+0000 mgr.y (mgr.25723) 562 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:23:54.260 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:54 smithi125 bash[80002]: cluster 2023-10-09T06:23:52.894736+0000 mgr.y (mgr.25723) 562 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:23:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:54.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.385 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:54.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:54 smithi155 bash[59495]: cluster 2023-10-09T06:23:52.894736+0000 mgr.y (mgr.25723) 562 : cluster 0 pgmap v533: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:23:54.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:54.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:54.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:54.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:54.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:54.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:54.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:55.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:55.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:55.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 53701 53685 108.221 81.1875 0.0101304 0.00923411 2023-10-09T06:23:55.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:55.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:55.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:23:55.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:55.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:55.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:55.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:55.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:55.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:55.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:55.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:55.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:55.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:55.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:56.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:56.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:56.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:56 smithi155 bash[59495]: cluster 2023-10-09T06:23:54.895524+0000 mgr.y (mgr.25723) 563 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:23:56.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 54948 54932 107.274 77.9375 0.00711817 0.00931543 2023-10-09T06:23:56.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:56.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:23:56.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:56 smithi125 bash[77299]: cluster 2023-10-09T06:23:54.895524+0000 mgr.y (mgr.25723) 563 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:23:56.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:56 smithi125 bash[80002]: cluster 2023-10-09T06:23:54.895524+0000 mgr.y (mgr.25723) 563 : cluster 0 pgmap v534: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:23:56.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:56.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:56.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:56.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:56.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:56.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:56.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:56.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:56.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:56.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:56.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:56.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:56.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:57.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:57.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:57.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 56171 56155 106.34 76.4375 0.0111424 0.00939853 2023-10-09T06:23:57.203 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:57.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:57.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:57.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:57.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:57.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:57.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:57.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:57.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:57.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:57.771 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:57.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:57.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:57.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:57.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:58.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:58.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:58.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:58.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:58 smithi155 bash[59495]: cluster 2023-10-09T06:23:56.896192+0000 mgr.y (mgr.25723) 564 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:23:58.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:58.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 57378 57362 105.431 75.4375 0.00767294 0.00947928 2023-10-09T06:23:58.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:58.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:58.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:58 smithi125 bash[77299]: cluster 2023-10-09T06:23:56.896192+0000 mgr.y (mgr.25723) 564 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:23:58.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:58 smithi125 bash[80002]: cluster 2023-10-09T06:23:56.896192+0000 mgr.y (mgr.25723) 564 : cluster 0 pgmap v535: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:23:58.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:58.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:58.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:58.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:58.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:58.789 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:58.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:58.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:58.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:58.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:59.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 58618 58602 104.632 77.5 0.011041 0.00955218 2023-10-09T06:23:59.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:59.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:59.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:59.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:59.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:59.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:59.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:59.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:59.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:59.726 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:23:59 smithi155 bash[59495]: cluster 2023-10-09T06:23:58.896828+0000 mgr.y (mgr.25723) 565 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:23:59.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:59.813 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:59.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:59.867 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:23:59.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:59.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:23:59 smithi125 bash[77299]: cluster 2023-10-09T06:23:58.896828+0000 mgr.y (mgr.25723) 565 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:23:59.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:23:59 smithi125 bash[80002]: cluster 2023-10-09T06:23:58.896828+0000 mgr.y (mgr.25723) 565 : cluster 0 pgmap v536: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:23:59.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:00.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:00.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:00.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:00.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:00.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 59863 59847 103.887 77.8125 0.0081383 0.00962056 2023-10-09T06:24:00.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:00.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:00.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:00.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:00.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:00.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:00.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:00.831 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:00.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:00.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:01.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:01.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:01.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:01.168 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 61100 61084 103.169 77.3125 0.00387735 0.00968648 2023-10-09T06:24:01.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:01.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:01.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:01.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:01.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:01.447 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:01] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-10-09T06:24:01.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:01.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:01.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:01.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:01.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:01.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:01.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:01.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:01.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:02.012 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:02.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:02.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:01 smithi125 bash[77299]: cluster 2023-10-09T06:24:00.897490+0000 mgr.y (mgr.25723) 566 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:24:02.130 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:01 smithi125 bash[77299]: audit 2023-10-09T06:24:01.786685+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:02.130 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:01 smithi125 bash[80002]: cluster 2023-10-09T06:24:00.897490+0000 mgr.y (mgr.25723) 566 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:24:02.130 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:01 smithi125 bash[80002]: audit 2023-10-09T06:24:01.786685+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:02.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:02.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:02.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 62220 62204 102.296 70 0.0134351 0.00977043 2023-10-09T06:24:02.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:02.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:01 smithi155 bash[59495]: cluster 2023-10-09T06:24:00.897490+0000 mgr.y (mgr.25723) 566 : cluster 0 pgmap v537: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:24:02.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:01 smithi155 bash[59495]: audit 2023-10-09T06:24:01.786685+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:02.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:02.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:02.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:02.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:02.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:02.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:02.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:02.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:02.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:02.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:03.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:03.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:03.169 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 63278 63262 101.368 66.125 0.00964246 0.00985865 2023-10-09T06:24:03.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:03.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:03.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:03.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:03.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[77299]: audit 2023-10-09T06:24:02.272060+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[77299]: audit 2023-10-09T06:24:02.282705+0000 mon.a (mon.0) 770 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[77299]: cluster 2023-10-09T06:24:02.898048+0000 mgr.y (mgr.25723) 567 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:24:03.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[80002]: audit 2023-10-09T06:24:02.272060+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[80002]: audit 2023-10-09T06:24:02.282705+0000 mon.a (mon.0) 770 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:03 smithi125 bash[80002]: cluster 2023-10-09T06:24:02.898048+0000 mgr.y (mgr.25723) 567 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:24:03.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:03 smithi155 bash[59495]: audit 2023-10-09T06:24:02.272060+0000 mon.a (mon.0) 769 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:03 smithi155 bash[59495]: audit 2023-10-09T06:24:02.282705+0000 mon.a (mon.0) 770 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:03.482 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:03 smithi155 bash[59495]: cluster 2023-10-09T06:24:02.898048+0000 mgr.y (mgr.25723) 567 : cluster 0 pgmap v538: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:24:03.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:03.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:03.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:03.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:03.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:03.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:03.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:04.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:04.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:04.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:24:04.164904+0000 min lat: 0.00129902 max lat: 0.219491 avg lat: 0.00994213 2023-10-09T06:24:04.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:24:04.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 64365 64349 100.53 67.9375 0.0151236 0.00994213 2023-10-09T06:24:04.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:04.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:04.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:04.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:04.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:04.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:04.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:04 smithi125 bash[77299]: audit 2023-10-09T06:24:03.578129+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:04.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:04 smithi125 bash[77299]: audit 2023-10-09T06:24:03.579511+0000 mon.a (mon.0) 772 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:04.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:04 smithi125 bash[80002]: audit 2023-10-09T06:24:03.578129+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:04.711 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:04 smithi125 bash[80002]: audit 2023-10-09T06:24:03.579511+0000 mon.a (mon.0) 772 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:04.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:04.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:04.844 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:04 smithi155 bash[59495]: audit 2023-10-09T06:24:03.578129+0000 mon.a (mon.0) 771 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:04.844 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:04 smithi155 bash[59495]: audit 2023-10-09T06:24:03.579511+0000 mon.a (mon.0) 772 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:04.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:04.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:04.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:05.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:05.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:05.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:05.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 65454 65438 99.7379 68.0625 0.00681647 0.0100209 2023-10-09T06:24:05.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.271 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:05.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:05.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:05.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:05.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:05.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:05.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:05.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:05.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:05.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:05.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:05 smithi125 bash[77299]: cluster 2023-10-09T06:24:04.898770+0000 mgr.y (mgr.25723) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:24:05.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:05 smithi125 bash[80002]: cluster 2023-10-09T06:24:04.898770+0000 mgr.y (mgr.25723) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:24:05.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:05 smithi155 bash[59495]: cluster 2023-10-09T06:24:04.898770+0000 mgr.y (mgr.25723) 568 : cluster 0 pgmap v539: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:24:05.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:05.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:06.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:06.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 66654 66638 99.1488 75 0.0147288 0.0100807 2023-10-09T06:24:06.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:06.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:06.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:06.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:06.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:06.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:06.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:06.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:06.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:06.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:06.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:06.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:06.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:06.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:07.008 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:07.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:07.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 67618 67602 98.2441 60.25 0.00964492 0.0101725 2023-10-09T06:24:07.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:07.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:07.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:07.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:07.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:07.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:07.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:07.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:07.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:07.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:07.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:08.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.051 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:07 smithi155 bash[59495]: cluster 2023-10-09T06:24:06.899235+0000 mgr.y (mgr.25723) 569 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:08.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 68734 68718 97.5964 69.75 0.0227011 0.0102407 2023-10-09T06:24:08.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:08.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:08.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:07 smithi125 bash[77299]: cluster 2023-10-09T06:24:06.899235+0000 mgr.y (mgr.25723) 569 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:24:08.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:07 smithi125 bash[80002]: cluster 2023-10-09T06:24:06.899235+0000 mgr.y (mgr.25723) 569 : cluster 0 pgmap v540: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:24:08.354 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:08.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:08.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:08.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:08.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:08.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:08.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:08.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:08.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:09.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:09.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:09.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:09.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 69803 69787 96.9105 66.8125 0.00931793 0.0103132 2023-10-09T06:24:09.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:09.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:09.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:09.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:09.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:09.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:09.799 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:09.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:09.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:09.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:09.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:10.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:10.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:09 smithi155 bash[59495]: audit 2023-10-09T06:24:08.753244+0000 mon.a (mon.0) 773 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:09 smithi155 bash[59495]: audit 2023-10-09T06:24:08.761534+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.078 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:09 smithi155 bash[59495]: cluster 2023-10-09T06:24:08.899830+0000 mgr.y (mgr.25723) 570 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:24:10.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[77299]: audit 2023-10-09T06:24:08.753244+0000 mon.a (mon.0) 773 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[77299]: audit 2023-10-09T06:24:08.761534+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[77299]: cluster 2023-10-09T06:24:08.899830+0000 mgr.y (mgr.25723) 570 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:24:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[80002]: audit 2023-10-09T06:24:08.753244+0000 mon.a (mon.0) 773 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[80002]: audit 2023-10-09T06:24:08.761534+0000 mon.a (mon.0) 774 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:09 smithi125 bash[80002]: cluster 2023-10-09T06:24:08.899830+0000 mgr.y (mgr.25723) 570 : cluster 0 pgmap v541: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:24:10.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 70891 70875 96.282 68 0.00410676 0.0103804 2023-10-09T06:24:10.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:10.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:10.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:10.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:10.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:10.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:10.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:10.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:10.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:10.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:10.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:10.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:10.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:11.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:11.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:11.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:11.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 72020 72004 95.7346 70.5625 0.013126 0.0104395 2023-10-09T06:24:11.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:11.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:11.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:11.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:11.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:11.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:11.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:11.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:11.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:11] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:24:11.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:11.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:11.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:11.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:11.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:11.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:12.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:12.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:12.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:12.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:11 smithi125 bash[77299]: cluster 2023-10-09T06:24:10.901199+0000 mgr.y (mgr.25723) 571 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:24:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:11 smithi125 bash[80002]: cluster 2023-10-09T06:24:10.901199+0000 mgr.y (mgr.25723) 571 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:24:12.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 73068 73052 95.1011 65.5 0.00906064 0.0105083 2023-10-09T06:24:12.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:12.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:12.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:12.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:11 smithi155 bash[59495]: cluster 2023-10-09T06:24:10.901199+0000 mgr.y (mgr.25723) 571 : cluster 0 pgmap v542: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:24:12.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:12.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:12.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:12.670 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:24:12.686 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:24:12.687 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:24:12.687 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:24:12.687 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:24:12.687 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:24:12.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:12.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:12.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:12.721 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:24:12.777 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:24:12.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:12.820 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:24:12.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:12.868 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:24:12.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:12.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:12.908 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:24:12.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:24:12.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:12.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:12 smithi155 bash[59495]: audit 2023-10-09T06:24:12.626721+0000 mon.a (mon.0) 775 : audit 0 from='client.? 172.21.15.125:0/1411958895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:13.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:24:13.023 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.023 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.024 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.025 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:24:13.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:13.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:13.063 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:12 smithi125 bash[77299]: audit 2023-10-09T06:24:12.626721+0000 mon.a (mon.0) 775 : audit 0 from='client.? 172.21.15.125:0/1411958895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:13.064 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:12 smithi125 bash[80002]: audit 2023-10-09T06:24:12.626721+0000 mon.a (mon.0) 775 : audit 0 from='client.? 172.21.15.125:0/1411958895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:13.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:13.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 73804 73788 94.099 46 0.00792457 0.0106214 2023-10-09T06:24:13.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:13.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:13.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:13.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:13.442 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:24:13.445 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:24:13.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:13.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:13.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:13.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:13.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:13.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:13.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:13.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:13.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:14.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: cluster 2023-10-09T06:24:12.901895+0000 mgr.y (mgr.25723) 572 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.141178+0000 mon.a (mon.0) 776 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.150743+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.543233+0000 mon.a (mon.0) 778 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.547686+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.125:0/4116575403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.785181+0000 mon.a (mon.0) 779 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.787411+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[77299]: audit 2023-10-09T06:24:13.795142+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: cluster 2023-10-09T06:24:12.901895+0000 mgr.y (mgr.25723) 572 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.141178+0000 mon.a (mon.0) 776 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.150743+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.543233+0000 mon.a (mon.0) 778 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.547686+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.125:0/4116575403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.785181+0000 mon.a (mon.0) 779 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:14.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.787411+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:14.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:13 smithi125 bash[80002]: audit 2023-10-09T06:24:13.795142+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: cluster 2023-10-09T06:24:12.901895+0000 mgr.y (mgr.25723) 572 : cluster 0 pgmap v543: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.141178+0000 mon.a (mon.0) 776 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.150743+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.543233+0000 mon.a (mon.0) 778 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.547686+0000 mon.b (mon.1) 8 : audit 1 from='client.? 172.21.15.125:0/4116575403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.785181+0000 mon.a (mon.0) 779 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.787411+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:13 smithi155 bash[59495]: audit 2023-10-09T06:24:13.795142+0000 mon.a (mon.0) 781 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:24:14.165 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:24:14.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 74733 74717 93.3783 58.0625 0.0229522 0.0107035 2023-10-09T06:24:14.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:14.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:14.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:14.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:14.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:14.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:14.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:14.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:14.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:15.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:15.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: audit 2023-10-09T06:24:13.410488+0000 mgr.y (mgr.25723) 573 : audit 0 from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:15.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.145993+0000 mon.a (mon.0) 782 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:15.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: audit 2023-10-09T06:24:14.153246+0000 mon.a (mon.0) 783 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:15.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.153903+0000 mon.a (mon.0) 784 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-09T06:24:15.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.157387+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.167613+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.168545+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:14 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.172229+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:15.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 75827 75811 92.8879 68.375 0.0111324 0.0107604 2023-10-09T06:24:15.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:15.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:15.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:15.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:15.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: audit 2023-10-09T06:24:13.410488+0000 mgr.y (mgr.25723) 573 : audit 0 from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:15.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.145993+0000 mon.a (mon.0) 782 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:15.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: audit 2023-10-09T06:24:14.153246+0000 mon.a (mon.0) 783 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:15.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.153903+0000 mon.a (mon.0) 784 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-09T06:24:15.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.157387+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.167613+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.168545+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.172229+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: audit 2023-10-09T06:24:13.410488+0000 mgr.y (mgr.25723) 573 : audit 0 from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.145993+0000 mon.a (mon.0) 782 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: audit 2023-10-09T06:24:14.153246+0000 mon.a (mon.0) 783 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.153903+0000 mon.a (mon.0) 784 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.157387+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.167613+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.168545+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:14 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.172229+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e148 with expected crc 2023-10-09T06:24:15.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:15.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:15.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:15.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:15.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:15.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:15.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:24:15.936 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":543,"stamp":"2023-10-09T06:24:12.901412+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4637071283,"num_objects":70711,"num_object_clones":0,"num_object_copies":212133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70711,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":71789,"num_write_kb":4538871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":1190420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71853,"ondisk_log_size":71853,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15913728,"kb_used_data":13981504,"kb_used_omap":30,"kb_used_meta":1932129,"kb_avail":734047488,"statfs":{"total":767960285184,"available":751664627712,"internally_reserved":0,"allocated":14317060096,"data_stored":14312356693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":1978500207},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":800391168,"num_objects":12213,"num_object_clones":0,"num_object_copies":36639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12213,"num_write_kb":781632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004380"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":128,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376843+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:23:21.376843+0000","last_peered":"2023-10-09T06:23:21.376843+0000","last_clean":"2023-10-09T06:23:21.376843+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:23:21.376843+0000","last_undegraded":"2023-10-09T06:23:21.376843+0000","last_fullsized":"2023-10-09T06:23:21.376843+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":118,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.891057+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:23:20.891057+0000","last_peered":"2023-10-09T06:23:20.891057+0000","last_clean":"2023-10-09T06:23:20.891057+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:23:20.891057+0000","last_undegraded":"2023-10-09T06:23:20.891057+0000","last_fullsized":"2023-10-09T06:23:20.891057+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":94,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.716785+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:23:21.716785+0000","last_peered":"2023-10-09T06:23:21.716785+0000","last_clean":"2023-10-09T06:23:21.716785+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:23:21.716785+0000","last_undegraded":"2023-10-09T06:23:21.716785+0000","last_fullsized":"2023-10-09T06:23:21.716785+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943315+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:23:20.943315+0000","last_peered":"2023-10-09T06:23:20.943315+0000","last_clean":"2023-10-09T06:23:20.943315+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:23:20.943315+0000","last_undegraded":"2023-10-09T06:23:20.943315+0000","last_fullsized":"2023-10-09T06:23:20.943315+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890456+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:23:20.890456+0000","last_peered":"2023-10-09T06:23:20.890456+0000","last_clean":"2023-10-09T06:23:20.890456+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:23:20.890456+0000","last_undegraded":"2023-10-09T06:23:20.890456+0000","last_fullsized":"2023-10-09T06:23:20.890456+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.900420+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:23:20.900420+0000","last_peered":"2023-10-09T06:23:20.900420+0000","last_clean":"2023-10-09T06:23:20.900420+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:23:20.900420+0000","last_undegraded":"2023-10-09T06:23:20.900420+0000","last_fullsized":"2023-10-09T06:23:20.900420+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943407+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:23:20.943407+0000","last_peered":"2023-10-09T06:23:20.943407+0000","last_clean":"2023-10-09T06:23:20.943407+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:23:20.943407+0000","last_undegraded":"2023-10-09T06:23:20.943407+0000","last_fullsized":"2023-10-09T06:23:20.943407+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":132,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.003517+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:23:52.003517+0000","last_peered":"2023-10-09T06:23:52.003517+0000","last_clean":"2023-10-09T06:23:52.003517+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:23:52.003517+0000","last_undegraded":"2023-10-09T06:23:52.003517+0000","last_fullsized":"2023-10-09T06:23:52.003517+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378829+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:23:21.378829+0000","last_peered":"2023-10-09T06:23:21.378829+0000","last_clean":"2023-10-09T06:23:21.378829+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:23:21.378829+0000","last_undegraded":"2023-10-09T06:23:21.378829+0000","last_fullsized":"2023-10-09T06:23:21.378829+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.002979+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:23:52.002979+0000","last_peered":"2023-10-09T06:23:52.002979+0000","last_clean":"2023-10-09T06:23:52.002979+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:23:52.002979+0000","last_undegraded":"2023-10-09T06:23:52.002979+0000","last_fullsized":"2023-10-09T06:23:52.002979+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":45,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026528+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:23:21.026528+0000","last_peered":"2023-10-09T06:23:21.026528+0000","last_clean":"2023-10-09T06:23:21.026528+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:23:21.026528+0000","last_undegraded":"2023-10-09T06:23:21.026528+0000","last_fullsized":"2023-10-09T06:23:21.026528+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":95,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.372040+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:23:21.372040+0000","last_peered":"2023-10-09T06:23:21.372040+0000","last_clean":"2023-10-09T06:23:21.372040+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:23:21.372040+0000","last_undegraded":"2023-10-09T06:23:21.372040+0000","last_fullsized":"2023-10-09T06:23:21.372040+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":157,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377459+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:23:21.377459+0000","last_peered":"2023-10-09T06:23:21.377459+0000","last_clean":"2023-10-09T06:23:21.377459+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:23:21.377459+0000","last_undegraded":"2023-10-09T06:23:21.377459+0000","last_fullsized":"2023-10-09T06:23:21.377459+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377330+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:23:21.377330+0000","last_peered":"2023-10-09T06:23:21.377330+0000","last_clean":"2023-10-09T06:23:21.377330+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:23:21.377330+0000","last_undegraded":"2023-10-09T06:23:21.377330+0000","last_fullsized":"2023-10-09T06:23:21.377330+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":45,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026477+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:23:21.026477+0000","last_peered":"2023-10-09T06:23:21.026477+0000","last_clean":"2023-10-09T06:23:21.026477+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:23:21.026477+0000","last_undegraded":"2023-10-09T06:23:21.026477+0000","last_fullsized":"2023-10-09T06:23:21.026477+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":137,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.003273+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:23:52.003273+0000","last_peered":"2023-10-09T06:23:52.003273+0000","last_clean":"2023-10-09T06:23:52.003273+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:23:52.003273+0000","last_undegraded":"2023-10-09T06:23:52.003273+0000","last_fullsized":"2023-10-09T06:23:52.003273+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943057+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:23:20.943057+0000","last_peered":"2023-10-09T06:23:20.943057+0000","last_clean":"2023-10-09T06:23:20.943057+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:23:20.943057+0000","last_undegraded":"2023-10-09T06:23:20.943057+0000","last_fullsized":"2023-10-09T06:23:20.943057+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378913+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:23:21.378913+0000","last_peered":"2023-10-09T06:23:21.378913+0000","last_clean":"2023-10-09T06:23:21.378913+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:23:21.378913+0000","last_undegraded":"2023-10-09T06:23:21.378913+0000","last_fullsized":"2023-10-09T06:23:21.378913+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":39,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.025818+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:23:21.025818+0000","last_peered":"2023-10-09T06:23:21.025818+0000","last_clean":"2023-10-09T06:23:21.025818+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:23:21.025818+0000","last_undegraded":"2023-10-09T06:23:21.025818+0000","last_fullsized":"2023-10-09T06:23:21.025818+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943264+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:23:20.943264+0000","last_peered":"2023-10-09T06:23:20.943264+0000","last_clean":"2023-10-09T06:23:20.943264+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:23:20.943264+0000","last_undegraded":"2023-10-09T06:23:20.943264+0000","last_fullsized":"2023-10-09T06:23:20.943264+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":121,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.899815+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:23:20.899815+0000","last_peered":"2023-10-09T06:23:20.899815+0000","last_clean":"2023-10-09T06:23:20.899815+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:23:20.899815+0000","last_undegraded":"2023-10-09T06:23:20.899815+0000","last_fullsized":"2023-10-09T06:23:20.899815+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.891226+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:23:20.891226+0000","last_peered":"2023-10-09T06:23:20.891226+0000","last_clean":"2023-10-09T06:23:20.891226+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:23:20.891226+0000","last_undegraded":"2023-10-09T06:23:20.891226+0000","last_fullsized":"2023-10-09T06:23:20.891226+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"147'4425","reported_seq":4438,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.368578+0000","last_change":"2023-10-09T06:23:02.818534+0000","last_active":"2023-10-09T06:24:09.368578+0000","last_peered":"2023-10-09T06:24:09.368578+0000","last_clean":"2023-10-09T06:24:09.368578+0000","last_became_active":"2023-10-09T06:23:02.816918+0000","last_became_peered":"2023-10-09T06:23:02.816918+0000","last_unstale":"2023-10-09T06:24:09.368578+0000","last_undegraded":"2023-10-09T06:24:09.368578+0000","last_fullsized":"2023-10-09T06:24:09.368578+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4425,"log_dups_size":0,"ondisk_log_size":4425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289996800,"num_objects":4425,"num_object_clones":0,"num_object_copies":13275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4425,"num_write_kb":283200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":43,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026742+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:23:21.026742+0000","last_peered":"2023-10-09T06:23:21.026742+0000","last_clean":"2023-10-09T06:23:21.026742+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:23:21.026742+0000","last_undegraded":"2023-10-09T06:23:21.026742+0000","last_fullsized":"2023-10-09T06:23:21.026742+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376314+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:23:21.376314+0000","last_peered":"2023-10-09T06:23:21.376314+0000","last_clean":"2023-10-09T06:23:21.376314+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:23:21.376314+0000","last_undegraded":"2023-10-09T06:23:21.376314+0000","last_fullsized":"2023-10-09T06:23:21.376314+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"147'4525","reported_seq":4539,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:10.846670+0000","last_change":"2023-10-09T06:23:02.810028+0000","last_active":"2023-10-09T06:24:10.846670+0000","last_peered":"2023-10-09T06:24:10.846670+0000","last_clean":"2023-10-09T06:24:10.846670+0000","last_became_active":"2023-10-09T06:23:02.809179+0000","last_became_peered":"2023-10-09T06:23:02.809179+0000","last_unstale":"2023-10-09T06:24:10.846670+0000","last_undegraded":"2023-10-09T06:24:10.846670+0000","last_fullsized":"2023-10-09T06:24:10.846670+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4525,"log_dups_size":0,"ondisk_log_size":4525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:49.851184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296550400,"num_objects":4525,"num_object_clones":0,"num_object_copies":13575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4525,"num_write_kb":289600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":43,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026742+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:23:21.026742+0000","last_peered":"2023-10-09T06:23:21.026742+0000","last_clean":"2023-10-09T06:23:21.026742+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:23:21.026742+0000","last_undegraded":"2023-10-09T06:23:21.026742+0000","last_fullsized":"2023-10-09T06:23:21.026742+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376503+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:23:21.376503+0000","last_peered":"2023-10-09T06:23:21.376503+0000","last_clean":"2023-10-09T06:23:21.376503+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:23:21.376503+0000","last_undegraded":"2023-10-09T06:23:21.376503+0000","last_fullsized":"2023-10-09T06:23:21.376503+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"147'4338","reported_seq":4349,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.443773+0000","last_change":"2023-10-09T06:23:02.818119+0000","last_active":"2023-10-09T06:24:09.443773+0000","last_peered":"2023-10-09T06:24:09.443773+0000","last_clean":"2023-10-09T06:24:09.443773+0000","last_became_active":"2023-10-09T06:23:02.817802+0000","last_became_peered":"2023-10-09T06:23:02.817802+0000","last_unstale":"2023-10-09T06:24:09.443773+0000","last_undegraded":"2023-10-09T06:24:09.443773+0000","last_fullsized":"2023-10-09T06:24:09.443773+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4338,"log_dups_size":0,"ondisk_log_size":4338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4338,"num_write_kb":277632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":43,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026668+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:23:21.026668+0000","last_peered":"2023-10-09T06:23:21.026668+0000","last_clean":"2023-10-09T06:23:21.026668+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:23:21.026668+0000","last_undegraded":"2023-10-09T06:23:21.026668+0000","last_fullsized":"2023-10-09T06:23:21.026668+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":130,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.899951+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:23:20.899951+0000","last_peered":"2023-10-09T06:23:20.899951+0000","last_clean":"2023-10-09T06:23:20.899951+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:23:20.899951+0000","last_undegraded":"2023-10-09T06:23:20.899951+0000","last_fullsized":"2023-10-09T06:23:20.899951+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"147'4237","reported_seq":4251,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:08.193991+0000","last_change":"2023-10-09T06:23:02.807021+0000","last_active":"2023-10-09T06:24:08.193991+0000","last_peered":"2023-10-09T06:24:08.193991+0000","last_clean":"2023-10-09T06:24:08.193991+0000","last_became_active":"2023-10-09T06:23:02.806707+0000","last_became_peered":"2023-10-09T06:23:02.806707+0000","last_unstale":"2023-10-09T06:24:08.193991+0000","last_undegraded":"2023-10-09T06:24:08.193991+0000","last_fullsized":"2023-10-09T06:24:08.193991+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4237,"log_dups_size":0,"ondisk_log_size":4237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:19.835325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277676032,"num_objects":4237,"num_object_clones":0,"num_object_copies":12711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4237,"num_write_kb":271168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":112,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378589+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:23:21.378589+0000","last_peered":"2023-10-09T06:23:21.378589+0000","last_clean":"2023-10-09T06:23:21.378589+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:23:21.378589+0000","last_undegraded":"2023-10-09T06:23:21.378589+0000","last_fullsized":"2023-10-09T06:23:21.378589+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":98,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943021+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:23:20.943021+0000","last_peered":"2023-10-09T06:23:20.943021+0000","last_clean":"2023-10-09T06:23:20.943021+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:23:20.943021+0000","last_undegraded":"2023-10-09T06:23:20.943021+0000","last_fullsized":"2023-10-09T06:23:20.943021+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"147'4363","reported_seq":4376,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.441859+0000","last_change":"2023-10-09T06:23:02.820523+0000","last_active":"2023-10-09T06:24:09.441859+0000","last_peered":"2023-10-09T06:24:09.441859+0000","last_clean":"2023-10-09T06:24:09.441859+0000","last_became_active":"2023-10-09T06:23:02.820107+0000","last_became_peered":"2023-10-09T06:23:02.820107+0000","last_unstale":"2023-10-09T06:24:09.441859+0000","last_undegraded":"2023-10-09T06:24:09.441859+0000","last_fullsized":"2023-10-09T06:24:09.441859+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4363,"log_dups_size":0,"ondisk_log_size":4363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285933568,"num_objects":4363,"num_object_clones":0,"num_object_copies":13089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4363,"num_write_kb":279232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:40.831556+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:23:40.831556+0000","last_peered":"2023-10-09T06:23:40.831556+0000","last_clean":"2023-10-09T06:23:40.831556+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:23:40.831556+0000","last_undegraded":"2023-10-09T06:23:40.831556+0000","last_fullsized":"2023-10-09T06:23:40.831556+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376410+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:23:21.376410+0000","last_peered":"2023-10-09T06:23:21.376410+0000","last_clean":"2023-10-09T06:23:21.376410+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:23:21.376410+0000","last_undegraded":"2023-10-09T06:23:21.376410+0000","last_fullsized":"2023-10-09T06:23:21.376410+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"147'4446","reported_seq":4460,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:11.543788+0000","last_change":"2023-10-09T06:23:02.812031+0000","last_active":"2023-10-09T06:24:11.543788+0000","last_peered":"2023-10-09T06:24:11.543788+0000","last_clean":"2023-10-09T06:24:11.543788+0000","last_became_active":"2023-10-09T06:23:02.811742+0000","last_became_peered":"2023-10-09T06:23:02.811742+0000","last_unstale":"2023-10-09T06:24:11.543788+0000","last_undegraded":"2023-10-09T06:24:11.543788+0000","last_fullsized":"2023-10-09T06:24:11.543788+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4446,"log_dups_size":0,"ondisk_log_size":4446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.791030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4446,"num_write_kb":284544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.900194+0000","last_change":"2023-10-09T06:13:43.443323+0000","last_active":"2023-10-09T06:23:20.900194+0000","last_peered":"2023-10-09T06:23:20.900194+0000","last_clean":"2023-10-09T06:23:20.900194+0000","last_became_active":"2023-10-09T06:13:43.443014+0000","last_became_peered":"2023-10-09T06:13:43.443014+0000","last_unstale":"2023-10-09T06:23:20.900194+0000","last_undegraded":"2023-10-09T06:23:20.900194+0000","last_fullsized":"2023-10-09T06:23:20.900194+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377773+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:23:21.377773+0000","last_peered":"2023-10-09T06:23:21.377773+0000","last_clean":"2023-10-09T06:23:21.377773+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:23:21.377773+0000","last_undegraded":"2023-10-09T06:23:21.377773+0000","last_fullsized":"2023-10-09T06:23:21.377773+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"147'4454","reported_seq":4468,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:10.852488+0000","last_change":"2023-10-09T06:23:02.813423+0000","last_active":"2023-10-09T06:24:10.852488+0000","last_peered":"2023-10-09T06:24:10.852488+0000","last_clean":"2023-10-09T06:24:10.852488+0000","last_became_active":"2023-10-09T06:23:02.813075+0000","last_became_peered":"2023-10-09T06:23:02.813075+0000","last_unstale":"2023-10-09T06:24:10.852488+0000","last_undegraded":"2023-10-09T06:24:10.852488+0000","last_fullsized":"2023-10-09T06:24:10.852488+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4454,"log_dups_size":0,"ondisk_log_size":4454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:07.790690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897344,"num_objects":4454,"num_object_clones":0,"num_object_copies":13362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4454,"num_write_kb":285056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376096+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:23:21.376096+0000","last_peered":"2023-10-09T06:23:21.376096+0000","last_clean":"2023-10-09T06:23:21.376096+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:23:21.376096+0000","last_undegraded":"2023-10-09T06:23:21.376096+0000","last_fullsized":"2023-10-09T06:23:21.376096+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377996+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:23:21.377996+0000","last_peered":"2023-10-09T06:23:21.377996+0000","last_clean":"2023-10-09T06:23:21.377996+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:23:21.377996+0000","last_undegraded":"2023-10-09T06:23:21.377996+0000","last_fullsized":"2023-10-09T06:23:21.377996+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"147'4341","reported_seq":4354,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.386135+0000","last_change":"2023-10-09T06:23:02.819776+0000","last_active":"2023-10-09T06:24:09.386135+0000","last_peered":"2023-10-09T06:24:09.386135+0000","last_clean":"2023-10-09T06:24:09.386135+0000","last_became_active":"2023-10-09T06:23:02.819408+0000","last_became_peered":"2023-10-09T06:23:02.819408+0000","last_unstale":"2023-10-09T06:24:09.386135+0000","last_undegraded":"2023-10-09T06:24:09.386135+0000","last_fullsized":"2023-10-09T06:24:09.386135+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4341,"log_dups_size":0,"ondisk_log_size":4341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284491776,"num_objects":4341,"num_object_clones":0,"num_object_copies":13023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4341,"num_write_kb":277824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":138,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890594+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:23:20.890594+0000","last_peered":"2023-10-09T06:23:20.890594+0000","last_clean":"2023-10-09T06:23:20.890594+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:23:20.890594+0000","last_undegraded":"2023-10-09T06:23:20.890594+0000","last_fullsized":"2023-10-09T06:23:20.890594+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.372283+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:23:21.372283+0000","last_peered":"2023-10-09T06:23:21.372283+0000","last_clean":"2023-10-09T06:23:21.372283+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:23:21.372283+0000","last_undegraded":"2023-10-09T06:23:21.372283+0000","last_fullsized":"2023-10-09T06:23:21.372283+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"147'4294","reported_seq":4308,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:08.199721+0000","last_change":"2023-10-09T06:23:02.808018+0000","last_active":"2023-10-09T06:24:08.199721+0000","last_peered":"2023-10-09T06:24:08.199721+0000","last_clean":"2023-10-09T06:24:08.199721+0000","last_became_active":"2023-10-09T06:23:02.807707+0000","last_became_peered":"2023-10-09T06:23:02.807707+0000","last_unstale":"2023-10-09T06:24:08.199721+0000","last_undegraded":"2023-10-09T06:24:08.199721+0000","last_fullsized":"2023-10-09T06:24:08.199721+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4294,"log_dups_size":0,"ondisk_log_size":4294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.598902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281411584,"num_objects":4294,"num_object_clones":0,"num_object_copies":12882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4294,"num_write_kb":274816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":996,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.007710+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:23:52.007710+0000","last_peered":"2023-10-09T06:23:52.007710+0000","last_clean":"2023-10-09T06:23:52.007710+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:23:52.007710+0000","last_undegraded":"2023-10-09T06:23:52.007710+0000","last_fullsized":"2023-10-09T06:23:52.007710+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378988+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:23:21.378988+0000","last_peered":"2023-10-09T06:23:21.378988+0000","last_clean":"2023-10-09T06:23:21.378988+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:23:21.378988+0000","last_undegraded":"2023-10-09T06:23:21.378988+0000","last_fullsized":"2023-10-09T06:23:21.378988+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.007031+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:23:52.007031+0000","last_peered":"2023-10-09T06:23:52.007031+0000","last_clean":"2023-10-09T06:23:52.007031+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:23:52.007031+0000","last_undegraded":"2023-10-09T06:23:52.007031+0000","last_fullsized":"2023-10-09T06:23:52.007031+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"147'4495","reported_seq":4509,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:10.821415+0000","last_change":"2023-10-09T06:23:02.809922+0000","last_active":"2023-10-09T06:24:10.821415+0000","last_peered":"2023-10-09T06:24:10.821415+0000","last_clean":"2023-10-09T06:24:10.821415+0000","last_became_active":"2023-10-09T06:23:02.809657+0000","last_became_peered":"2023-10-09T06:23:02.809657+0000","last_unstale":"2023-10-09T06:24:10.821415+0000","last_undegraded":"2023-10-09T06:24:10.821415+0000","last_fullsized":"2023-10-09T06:24:10.821415+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4495,"log_dups_size":0,"ondisk_log_size":4495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:22.193166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294584320,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4495,"num_write_kb":287680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":728,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377553+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:23:21.377553+0000","last_peered":"2023-10-09T06:23:21.377553+0000","last_clean":"2023-10-09T06:23:21.377553+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:23:21.377553+0000","last_undegraded":"2023-10-09T06:23:21.377553+0000","last_fullsized":"2023-10-09T06:23:21.377553+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":112,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377056+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:23:21.377056+0000","last_peered":"2023-10-09T06:23:21.377056+0000","last_clean":"2023-10-09T06:23:21.377056+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:23:21.377056+0000","last_undegraded":"2023-10-09T06:23:21.377056+0000","last_fullsized":"2023-10-09T06:23:21.377056+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.007572+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:23:52.007572+0000","last_peered":"2023-10-09T06:23:52.007572+0000","last_clean":"2023-10-09T06:23:52.007572+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:23:52.007572+0000","last_undegraded":"2023-10-09T06:23:52.007572+0000","last_fullsized":"2023-10-09T06:23:52.007572+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"147'4316","reported_seq":4328,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.442365+0000","last_change":"2023-10-09T06:23:02.818151+0000","last_active":"2023-10-09T06:24:09.442365+0000","last_peered":"2023-10-09T06:24:09.442365+0000","last_clean":"2023-10-09T06:24:09.442365+0000","last_became_active":"2023-10-09T06:23:02.816332+0000","last_became_peered":"2023-10-09T06:23:02.816332+0000","last_unstale":"2023-10-09T06:24:09.442365+0000","last_undegraded":"2023-10-09T06:24:09.442365+0000","last_fullsized":"2023-10-09T06:24:09.442365+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4316,"log_dups_size":0,"ondisk_log_size":4316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":276224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":506,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.370788+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:23:21.370788+0000","last_peered":"2023-10-09T06:23:21.370788+0000","last_clean":"2023-10-09T06:23:21.370788+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:23:21.370788+0000","last_undegraded":"2023-10-09T06:23:21.370788+0000","last_fullsized":"2023-10-09T06:23:21.370788+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:41.725694+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:23:41.725694+0000","last_peered":"2023-10-09T06:23:41.725694+0000","last_clean":"2023-10-09T06:23:41.725694+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:23:41.725694+0000","last_undegraded":"2023-10-09T06:23:41.725694+0000","last_fullsized":"2023-10-09T06:23:41.725694+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.372301+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:23:21.372301+0000","last_peered":"2023-10-09T06:23:21.372301+0000","last_clean":"2023-10-09T06:23:21.372301+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:23:21.372301+0000","last_undegraded":"2023-10-09T06:23:21.372301+0000","last_fullsized":"2023-10-09T06:23:21.372301+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"147'4402","reported_seq":4415,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.421175+0000","last_change":"2023-10-09T06:23:02.821017+0000","last_active":"2023-10-09T06:24:09.421175+0000","last_peered":"2023-10-09T06:24:09.421175+0000","last_clean":"2023-10-09T06:24:09.421175+0000","last_became_active":"2023-10-09T06:23:02.820708+0000","last_became_peered":"2023-10-09T06:23:02.820708+0000","last_unstale":"2023-10-09T06:24:09.421175+0000","last_undegraded":"2023-10-09T06:24:09.421175+0000","last_fullsized":"2023-10-09T06:24:09.421175+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4402,"log_dups_size":0,"ondisk_log_size":4402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288489472,"num_objects":4402,"num_object_clones":0,"num_object_copies":13206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4402,"num_write_kb":281728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":495,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378472+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:23:21.378472+0000","last_peered":"2023-10-09T06:23:21.378472+0000","last_clean":"2023-10-09T06:23:21.378472+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:23:21.378472+0000","last_undegraded":"2023-10-09T06:23:21.378472+0000","last_fullsized":"2023-10-09T06:23:21.378472+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":343,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.025928+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:23:21.025928+0000","last_peered":"2023-10-09T06:23:21.025928+0000","last_clean":"2023-10-09T06:23:21.025928+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:23:21.025928+0000","last_undegraded":"2023-10-09T06:23:21.025928+0000","last_fullsized":"2023-10-09T06:23:21.025928+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890445+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:23:20.890445+0000","last_peered":"2023-10-09T06:23:20.890445+0000","last_clean":"2023-10-09T06:23:20.890445+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:23:20.890445+0000","last_undegraded":"2023-10-09T06:23:20.890445+0000","last_fullsized":"2023-10-09T06:23:20.890445+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.900410+0000","last_change":"2023-10-09T06:18:59.807622+0000","last_active":"2023-10-09T06:23:20.900410+0000","last_peered":"2023-10-09T06:23:20.900410+0000","last_clean":"2023-10-09T06:23:20.900410+0000","last_became_active":"2023-10-09T06:18:59.807242+0000","last_became_peered":"2023-10-09T06:18:59.807242+0000","last_unstale":"2023-10-09T06:23:20.900410+0000","last_undegraded":"2023-10-09T06:23:20.900410+0000","last_fullsized":"2023-10-09T06:23:20.900410+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"147'4636","reported_seq":4650,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:11.740993+0000","last_change":"2023-10-09T06:23:02.811831+0000","last_active":"2023-10-09T06:24:11.740993+0000","last_peered":"2023-10-09T06:24:11.740993+0000","last_clean":"2023-10-09T06:24:11.740993+0000","last_became_active":"2023-10-09T06:23:02.811445+0000","last_became_peered":"2023-10-09T06:23:02.811445+0000","last_unstale":"2023-10-09T06:24:11.740993+0000","last_undegraded":"2023-10-09T06:24:11.740993+0000","last_fullsized":"2023-10-09T06:24:11.740993+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:28.021647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303824896,"num_objects":4636,"num_object_clones":0,"num_object_copies":13908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4636,"num_write_kb":296704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":68,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.025470+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:23:21.025470+0000","last_peered":"2023-10-09T06:23:21.025470+0000","last_clean":"2023-10-09T06:23:21.025470+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:23:21.025470+0000","last_undegraded":"2023-10-09T06:23:21.025470+0000","last_fullsized":"2023-10-09T06:23:21.025470+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:38.195095+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:23:38.195095+0000","last_peered":"2023-10-09T06:23:38.195095+0000","last_clean":"2023-10-09T06:23:38.195095+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:23:38.195095+0000","last_undegraded":"2023-10-09T06:23:38.195095+0000","last_fullsized":"2023-10-09T06:23:38.195095+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":114,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.008211+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:23:52.008211+0000","last_peered":"2023-10-09T06:23:52.008211+0000","last_clean":"2023-10-09T06:23:52.008211+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:23:52.008211+0000","last_undegraded":"2023-10-09T06:23:52.008211+0000","last_fullsized":"2023-10-09T06:23:52.008211+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"147'4523","reported_seq":4537,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:11.684682+0000","last_change":"2023-10-09T06:23:02.810207+0000","last_active":"2023-10-09T06:24:11.684682+0000","last_peered":"2023-10-09T06:24:11.684682+0000","last_clean":"2023-10-09T06:24:11.684682+0000","last_became_active":"2023-10-09T06:23:02.809844+0000","last_became_peered":"2023-10-09T06:23:02.809844+0000","last_unstale":"2023-10-09T06:24:11.684682+0000","last_undegraded":"2023-10-09T06:24:11.684682+0000","last_fullsized":"2023-10-09T06:24:11.684682+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4523,"log_dups_size":0,"ondisk_log_size":4523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:38.667224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296419328,"num_objects":4523,"num_object_clones":0,"num_object_copies":13569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4523,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":57,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.026473+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:23:21.026473+0000","last_peered":"2023-10-09T06:23:21.026473+0000","last_clean":"2023-10-09T06:23:21.026473+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:23:21.026473+0000","last_undegraded":"2023-10-09T06:23:21.026473+0000","last_fullsized":"2023-10-09T06:23:21.026473+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":95,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.007644+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:23:52.007644+0000","last_peered":"2023-10-09T06:23:52.007644+0000","last_clean":"2023-10-09T06:23:52.007644+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:23:52.007644+0000","last_undegraded":"2023-10-09T06:23:52.007644+0000","last_fullsized":"2023-10-09T06:23:52.007644+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:38.195165+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:23:38.195165+0000","last_peered":"2023-10-09T06:23:38.195165+0000","last_clean":"2023-10-09T06:23:38.195165+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:23:38.195165+0000","last_undegraded":"2023-10-09T06:23:38.195165+0000","last_fullsized":"2023-10-09T06:23:38.195165+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"147'4430","reported_seq":4443,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.417239+0000","last_change":"2023-10-09T06:23:02.821182+0000","last_active":"2023-10-09T06:24:09.417239+0000","last_peered":"2023-10-09T06:24:09.417239+0000","last_clean":"2023-10-09T06:24:09.417239+0000","last_became_active":"2023-10-09T06:23:02.820839+0000","last_became_peered":"2023-10-09T06:23:02.820839+0000","last_unstale":"2023-10-09T06:24:09.417239+0000","last_undegraded":"2023-10-09T06:24:09.417239+0000","last_fullsized":"2023-10-09T06:24:09.417239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4430,"log_dups_size":0,"ondisk_log_size":4430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4430,"num_write_kb":283520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":144,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.891303+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:23:20.891303+0000","last_peered":"2023-10-09T06:23:20.891303+0000","last_clean":"2023-10-09T06:23:20.891303+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:23:20.891303+0000","last_undegraded":"2023-10-09T06:23:20.891303+0000","last_fullsized":"2023-10-09T06:23:20.891303+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":451,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.372334+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:23:21.372334+0000","last_peered":"2023-10-09T06:23:21.372334+0000","last_clean":"2023-10-09T06:23:21.372334+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:23:21.372334+0000","last_undegraded":"2023-10-09T06:23:21.372334+0000","last_fullsized":"2023-10-09T06:23:21.372334+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890607+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:23:20.890607+0000","last_peered":"2023-10-09T06:23:20.890607+0000","last_clean":"2023-10-09T06:23:20.890607+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:23:20.890607+0000","last_undegraded":"2023-10-09T06:23:20.890607+0000","last_fullsized":"2023-10-09T06:23:20.890607+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"147'4458","reported_seq":4471,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:24:09.428392+0000","last_change":"2023-10-09T06:23:02.819624+0000","last_active":"2023-10-09T06:24:09.428392+0000","last_peered":"2023-10-09T06:24:09.428392+0000","last_clean":"2023-10-09T06:24:09.428392+0000","last_became_active":"2023-10-09T06:23:02.818630+0000","last_became_peered":"2023-10-09T06:23:02.818630+0000","last_unstale":"2023-10-09T06:24:09.428392+0000","last_undegraded":"2023-10-09T06:24:09.428392+0000","last_fullsized":"2023-10-09T06:24:09.428392+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":4458,"log_dups_size":0,"ondisk_log_size":4458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292159488,"num_objects":4458,"num_object_clones":0,"num_object_copies":13374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4458,"num_write_kb":285312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":843,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.900440+0000","last_change":"2023-10-09T06:22:46.890972+0000","last_active":"2023-10-09T06:23:20.900440+0000","last_peered":"2023-10-09T06:23:20.900440+0000","last_clean":"2023-10-09T06:23:20.900440+0000","last_became_active":"2023-10-09T06:22:46.889916+0000","last_became_peered":"2023-10-09T06:22:46.889916+0000","last_unstale":"2023-10-09T06:23:20.900440+0000","last_undegraded":"2023-10-09T06:23:20.900440+0000","last_fullsized":"2023-10-09T06:23:20.900440+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":103,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.008122+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:23:52.008122+0000","last_peered":"2023-10-09T06:23:52.008122+0000","last_clean":"2023-10-09T06:23:52.008122+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:23:52.008122+0000","last_undegraded":"2023-10-09T06:23:52.008122+0000","last_fullsized":"2023-10-09T06:23:52.008122+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376560+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:23:21.376560+0000","last_peered":"2023-10-09T06:23:21.376560+0000","last_clean":"2023-10-09T06:23:21.376560+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:23:21.376560+0000","last_undegraded":"2023-10-09T06:23:21.376560+0000","last_fullsized":"2023-10-09T06:23:21.376560+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":105,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.943262+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:23:20.943262+0000","last_peered":"2023-10-09T06:23:20.943262+0000","last_clean":"2023-10-09T06:23:20.943262+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:23:20.943262+0000","last_undegraded":"2023-10-09T06:23:20.943262+0000","last_fullsized":"2023-10-09T06:23:20.943262+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-10-09T06:23:52.010097+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:23:52.010097+0000","last_peered":"2023-10-09T06:23:52.010097+0000","last_clean":"2023-10-09T06:23:52.010097+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:23:52.010097+0000","last_undegraded":"2023-10-09T06:23:52.010097+0000","last_fullsized":"2023-10-09T06:23:52.010097+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.378876+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:23:21.378876+0000","last_peered":"2023-10-09T06:23:21.378876+0000","last_clean":"2023-10-09T06:23:21.378876+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:23:21.378876+0000","last_undegraded":"2023-10-09T06:23:21.378876+0000","last_fullsized":"2023-10-09T06:23:21.378876+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377508+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:23:21.377508+0000","last_peered":"2023-10-09T06:23:21.377508+0000","last_clean":"2023-10-09T06:23:21.377508+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:23:21.377508+0000","last_undegraded":"2023-10-09T06:23:21.377508+0000","last_fullsized":"2023-10-09T06:23:21.377508+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":99,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.379127+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:23:21.379127+0000","last_peered":"2023-10-09T06:23:21.379127+0000","last_clean":"2023-10-09T06:23:21.379127+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:23:21.379127+0000","last_undegraded":"2023-10-09T06:23:21.379127+0000","last_fullsized":"2023-10-09T06:23:21.379127+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.377208+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:23:21.377208+0000","last_peered":"2023-10-09T06:23:21.377208+0000","last_clean":"2023-10-09T06:23:21.377208+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:23:21.377208+0000","last_undegraded":"2023-10-09T06:23:21.377208+0000","last_fullsized":"2023-10-09T06:23:21.377208+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":125,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.891110+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:23:20.891110+0000","last_peered":"2023-10-09T06:23:20.891110+0000","last_clean":"2023-10-09T06:23:20.891110+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:23:20.891110+0000","last_undegraded":"2023-10-09T06:23:20.891110+0000","last_fullsized":"2023-10-09T06:23:20.891110+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:20.890336+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:23:20.890336+0000","last_peered":"2023-10-09T06:23:20.890336+0000","last_clean":"2023-10-09T06:23:20.890336+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:23:20.890336+0000","last_undegraded":"2023-10-09T06:23:20.890336+0000","last_fullsized":"2023-10-09T06:23:20.890336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":103,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:38.195228+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:23:38.195228+0000","last_peered":"2023-10-09T06:23:38.195228+0000","last_clean":"2023-10-09T06:23:38.195228+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:23:38.195228+0000","last_undegraded":"2023-10-09T06:23:38.195228+0000","last_fullsized":"2023-10-09T06:23:38.195228+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":111,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-10-09T06:23:21.376054+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:23:21.376054+0000","last_peered":"2023-10-09T06:23:21.376054+0000","last_clean":"2023-10-09T06:23:21.376054+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:23:21.376054+0000","last_undegraded":"2023-10-09T06:23:21.376054+0000","last_fullsized":"2023-10-09T06:23:21.376054+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4632281088,"num_objects":70683,"num_object_clones":0,"num_object_copies":212049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":70683,"num_write_kb":4523712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13961854976,"data_stored":13961854976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70683,"ondisk_log_size":70683,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430514,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1987660,"kb_used_data":1678148,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91757492,"statfs":{"total":95995035648,"available":93959671808,"internally_reserved":0,"allocated":1718423552,"data_stored":1717897175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.4319999999999999,"15min":1.3979999999999999},"min":{"1min":0.44800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.5129999999999999,"5min":107.36799999999999,"15min":107.36799999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.4209999999999998,"15min":1.387},"min":{"1min":0.49399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.131,"5min":105.89,"15min":105.89},"last":2.073}]},{"osd":1,"last update":"Mon Oct 9 06:24:03 2023","interfaces":[{"interface":"back","average":{"1min":3.6230000000000002,"5min":3.4140000000000001,"15min":1.7350000000000001},"min":{"1min":0.29599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":63.252000000000002,"5min":104.825,"15min":104.825},"last":0.37},{"interface":"front","average":{"1min":3.5299999999999998,"5min":2.952,"15min":1.6180000000000001},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.314},"max":{"1min":60.746000000000002,"5min":104.637,"15min":104.637},"last":1.288}]},{"osd":2,"last update":"Mon Oct 9 06:23:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":4.1470000000000002,"15min":1.998},"min":{"1min":0.52800000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.093,"5min":104.992,"15min":104.992},"last":1.224},{"interface":"front","average":{"1min":1.0009999999999999,"5min":4.0369999999999999,"15min":2.0129999999999999},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.3150000000000004,"5min":104.691,"15min":104.691},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":8.8000000000000007,"5min":3.0489999999999999,"15min":1.6739999999999999},"min":{"1min":0.56899999999999995,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":137.11600000000001,"5min":137.11600000000001,"15min":137.11600000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":8.875,"5min":2.7490000000000001,"15min":1.6659999999999999},"min":{"1min":0.495,"5min":0.38,"15min":0.38},"max":{"1min":137.14400000000001,"5min":137.14400000000001,"15min":137.14400000000001},"last":0.48999999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.7909999999999999,"15min":1.204},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29399999999999998},"max":{"1min":1.2470000000000001,"5min":11.029999999999999,"15min":11.029999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.87,"5min":1.948,"15min":1.276},"min":{"1min":0.52900000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.296,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":0.77200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.8280000000000001,"15min":1.3120000000000001},"min":{"1min":0.41499999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.4009999999999998,"5min":11.996,"15min":11.996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.752,"15min":1.2949999999999999},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.327,"5min":13.634,"15min":13.634},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.3939999999999999,"15min":1.1839999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3859999999999999,"5min":9.9510000000000005,"15min":9.9510000000000005},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.669,"15min":1.2809999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3400000000000001,"5min":8.6199999999999992,"15min":8.6199999999999992},"last":4.2110000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626746,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693052,"kb_used_data":1391732,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92052100,"statfs":{"total":95995035648,"available":94261350400,"internally_reserved":0,"allocated":1425133568,"data_stored":1424622818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.7,"15min":1.161},"min":{"1min":0.29199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.1309999999999998,"5min":32.289999999999999,"15min":32.289999999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.573,"15min":1.1379999999999999},"min":{"1min":0.45600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.6699999999999999,"5min":31.454999999999998,"15min":31.454999999999998},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":2.5779999999999998,"15min":1.4399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.242,"5min":137.93899999999999,"15min":137.93899999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.819,"15min":1.5329999999999999},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2050000000000001,"5min":138.02500000000001,"15min":138.02500000000001},"last":1.4910000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.98,"15min":1.639},"min":{"1min":0.63100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.4749999999999996,"5min":95.540000000000006,"15min":95.540000000000006},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.9580000000000002,"15min":1.6499999999999999},"min":{"1min":0.5,"5min":0.309,"15min":0.309},"max":{"1min":1.343,"5min":96.352000000000004,"15min":96.352000000000004},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":1.6559999999999999,"15min":1.242},"min":{"1min":0.502,"5min":0.316,"15min":0.316},"max":{"1min":23.315000000000001,"5min":23.315000000000001,"15min":23.315000000000001},"last":3.8759999999999999},{"interface":"front","average":{"1min":1.905,"5min":1.323,"15min":1.1060000000000001},"min":{"1min":0.36599999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":23.274999999999999,"5min":23.274999999999999,"15min":23.274999999999999},"last":3.4279999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.599,"15min":1.1839999999999999},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.3959999999999999,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.496},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.6890000000000001,"15min":1.2509999999999999},"min":{"1min":0.36799999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.2570000000000001,"5min":10.300000000000001,"15min":10.300000000000001},"last":0.36799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.907,"15min":1.3260000000000001},"min":{"1min":0.40799999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.0609999999999999,"5min":11.754,"15min":11.754},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.274,"5min":1.7809999999999999,"15min":1.321},"min":{"1min":0.40100000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.7210000000000001,"5min":12.295999999999999,"15min":12.295999999999999},"last":0.62}]},{"osd":7,"last update":"Mon Oct 9 06:23:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.7549999999999999,"15min":1.3220000000000001},"min":{"1min":0.53200000000000003,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.5900000000000001,"5min":9.4179999999999993,"15min":9.4179999999999993},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.9299999999999999,"15min":1.401},"min":{"1min":0.49299999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.3479999999999999,"5min":11.102,"15min":11.102},"last":0.46500000000000002}]}]},{"osd":1,"up_from":93,"seq":399431958656,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595300,"kb_used_data":1447132,"kb_used_omap":7,"kb_used_meta":148152,"kb_avail":92149852,"statfs":{"total":95995035648,"available":94361448448,"internally_reserved":0,"allocated":1481863168,"data_stored":1481201207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":151708048},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.3640000000000001,"15min":0.95099999999999996},"min":{"1min":0.20100000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":3.9550000000000001,"5min":31.969000000000001,"15min":31.969000000000001},"last":0.432},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.7669999999999999,"15min":1.087},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.5710000000000002,"5min":40.667999999999999,"15min":40.667999999999999},"last":0.24399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:23:36 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":1.2410000000000001,"15min":0.90200000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.4220000000000002,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":5.7549999999999999},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.278,"15min":0.999},"min":{"1min":0.40600000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.208,"5min":8.3960000000000008,"15min":8.3960000000000008},"last":0.628}]},{"osd":3,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":6.0119999999999996,"5min":2.117,"15min":1.3340000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":103.217,"5min":103.217,"15min":103.217},"last":0.59599999999999997},{"interface":"front","average":{"1min":6.4770000000000003,"5min":2.089,"15min":1.264},"min":{"1min":0.42199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":103.303,"5min":103.303,"15min":103.303},"last":2.371}]},{"osd":4,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.4990000000000001,"15min":1.165},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.3639999999999999,"5min":6.7469999999999999,"15min":6.7469999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.6639999999999999,"15min":1.2110000000000001},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.9159999999999999,"5min":7.3639999999999999,"15min":7.3639999999999999},"last":0.86899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.716,"15min":1.278},"min":{"1min":0.42599999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.1429999999999998,"5min":8.8049999999999997,"15min":8.8049999999999997},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.766,"15min":1.3049999999999999},"min":{"1min":0.38300000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.9550000000000001,"5min":6.7960000000000003,"15min":6.7960000000000003},"last":0.68400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.3620000000000001,"15min":1.1819999999999999},"min":{"1min":0.44800000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.6080000000000001,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.444,"15min":1.2390000000000001},"min":{"1min":0.40400000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.8069999999999999,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":5.8040000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":2.044,"15min":1.4319999999999999},"min":{"1min":0.5,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.3100000000000001,"5min":7.46,"15min":7.46},"last":1.873},{"interface":"front","average":{"1min":1.29,"5min":1.726,"15min":1.323},"min":{"1min":0.46100000000000002,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":4.0270000000000001,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":0.46600000000000003}]}]},{"osd":0,"up_from":88,"seq":377957122181,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628592,"kb_used_data":1394280,"kb_used_omap":8,"kb_used_meta":234295,"kb_avail":92116560,"statfs":{"total":95995035648,"available":94327357440,"internally_reserved":0,"allocated":1427742720,"data_stored":1427069332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":239918336},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:24:07 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.6830000000000001,"15min":1.03},"min":{"1min":0.23699999999999999,"5min":0.218,"15min":0.218},"max":{"1min":8.0310000000000006,"5min":12.460000000000001,"15min":12.460000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.6850000000000001,"15min":1.0860000000000001},"min":{"1min":0.29099999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.0619999999999994,"5min":8.9640000000000004,"15min":10.243},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:23:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.5109999999999999,"15min":0.95099999999999996},"min":{"1min":0.28399999999999997,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.0860000000000001,"5min":13.035,"15min":13.035},"last":0.20999999999999999},{"interface":"front","average":{"1min":0.622,"5min":1.7030000000000001,"15min":1.0569999999999999},"min":{"1min":0.33900000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.95999999999999996,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.56799999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.698,"5min":1.5509999999999999,"15min":1.0569999999999999},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":16.978000000000002,"5min":16.978000000000002,"15min":16.978000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.395,"5min":1.405,"15min":0.97699999999999998},"min":{"1min":0.30299999999999999,"5min":0.27300000000000002,"15min":0.26600000000000001},"max":{"1min":16.489000000000001,"5min":16.489000000000001,"15min":16.489000000000001},"last":1.5880000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.762,"15min":1.2370000000000001},"min":{"1min":0.443,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.3630000000000004,"5min":9.0890000000000004,"15min":9.0890000000000004},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.889,"15min":1.3169999999999999},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":6.0650000000000004,"5min":10.505000000000001,"15min":10.505000000000001},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.1259999999999999,"15min":1.415},"min":{"1min":0.45500000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.3390000000000004,"5min":9.548,"15min":9.548},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.8140000000000001,"15min":1.3109999999999999},"min":{"1min":0.58199999999999996,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.6360000000000001,"5min":10.329000000000001,"15min":10.329000000000001},"last":2.0510000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.548,"15min":1.2250000000000001},"min":{"1min":0.47699999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6589999999999998,"5min":10.194000000000001,"15min":10.194000000000001},"last":1.6259999999999999},{"interface":"front","average":{"1min":1.097,"5min":1.512,"15min":1.1970000000000001},"min":{"1min":0.496,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.4580000000000002,"5min":8.7629999999999999,"15min":8.7629999999999999},"last":0.67500000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.7190000000000001,"15min":1.333},"min":{"1min":0.60199999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.7480000000000002,"5min":6.0330000000000004,"15min":6.0330000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.9850000000000001,"15min":1.4179999999999999},"min":{"1min":0.51200000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":5.3849999999999998,"5min":8.7959999999999994,"15min":8.7959999999999994},"last":0.58199999999999996}]}]},{"osd":2,"up_from":98,"seq":420906795130,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2482012,"kb_used_data":2297812,"kb_used_omap":11,"kb_used_meta":184180,"kb_avail":91263140,"statfs":{"total":95995035648,"available":93453455360,"internally_reserved":0,"allocated":2352959488,"data_stored":2352301307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":188601278},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":2.1880000000000002,"15min":1.198},"min":{"1min":0.29599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":8.609,"5min":35.015999999999998,"15min":35.015999999999998},"last":1.3680000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":2.411,"15min":1.262},"min":{"1min":0.32800000000000001,"5min":0.182,"15min":0.182},"max":{"1min":6.7469999999999999,"5min":36.415999999999997,"15min":36.415999999999997},"last":0.56399999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.395,"15min":0.95299999999999996},"min":{"1min":0.40699999999999997,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.5399999999999991,"5min":8.5399999999999991,"15min":8.5399999999999991},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.9650000000000001,"15min":1.143},"min":{"1min":0.34999999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.0510000000000002,"5min":10.4,"15min":10.4},"last":0.95099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.486,"15min":1.006},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.0250000000000004,"5min":13.311999999999999,"15min":13.311999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.5089999999999999,"15min":0.96299999999999997},"min":{"1min":0.46899999999999997,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.2490000000000006,"5min":18.831,"15min":18.831},"last":1.766}]},{"osd":4,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":2.5760000000000001,"15min":1.51},"min":{"1min":0.57199999999999995,"5min":0.315,"15min":0.315},"max":{"1min":3.1179999999999999,"5min":19.184000000000001,"15min":19.184000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.2090000000000001,"5min":2.2869999999999999,"15min":1.3819999999999999},"min":{"1min":0.63300000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.2130000000000001,"5min":29.408000000000001,"15min":29.408000000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.2330000000000001,"15min":1.4239999999999999},"min":{"1min":0.53000000000000003,"5min":0.31,"15min":0.31},"max":{"1min":3.0819999999999999,"5min":12.833,"15min":12.833},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.159,"5min":2.0350000000000001,"15min":1.3089999999999999},"min":{"1min":0.62,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.96,"5min":29.545000000000002,"15min":29.545000000000002},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.605,"15min":1.2010000000000001},"min":{"1min":0.55100000000000005,"5min":0.372,"15min":0.372},"max":{"1min":3.8860000000000001,"5min":29.504000000000001,"15min":29.504000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.637,"5min":1.7549999999999999,"15min":1.2769999999999999},"min":{"1min":0.53600000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":7.0970000000000004,"5min":8.1349999999999998,"15min":8.1349999999999998},"last":0.41799999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.8879999999999999,"15min":1.377},"min":{"1min":0.46700000000000003,"5min":0.308,"15min":0.308},"max":{"1min":7.165,"5min":8.3309999999999995,"15min":8.3309999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.1859999999999999,"5min":2.3010000000000002,"15min":1.5},"min":{"1min":0.56000000000000005,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.2229999999999999,"5min":10.978999999999999,"15min":10.978999999999999},"last":0.878}]}]},{"osd":3,"up_from":103,"seq":442381631604,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2984788,"kb_used_data":2836812,"kb_used_omap":3,"kb_used_meta":147964,"kb_avail":90760364,"statfs":{"total":95995035648,"available":92938612736,"internally_reserved":0,"allocated":2904895488,"data_stored":2904244727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":151515681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.9279999999999999,"15min":1.113},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":2.1749999999999998,"5min":62.939,"15min":62.939},"last":3.2029999999999998},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.1499999999999999,"15min":1.1799999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":6.4950000000000001,"5min":63.869,"15min":63.869},"last":0.75900000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.115,"15min":0.84299999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.564,"5min":10.564,"15min":10.564},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.0329999999999999,"15min":0.875},"min":{"1min":0.34399999999999997,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":3.1309999999999998,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":1.1220000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.2490000000000001,"15min":1.3149999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2910000000000004,"5min":70.594999999999999,"15min":70.594999999999999},"last":1.196},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.9870000000000001,"15min":1.155},"min":{"1min":0.27500000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.5570000000000004,"5min":62.991999999999997,"15min":62.991999999999997},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.8600000000000001,"15min":1.2330000000000001},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.314},"max":{"1min":3.778,"5min":10.446999999999999,"15min":10.446999999999999},"last":2.2669999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.708,"15min":1.224},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.8199999999999998,"5min":7.8940000000000001,"15min":7.8940000000000001},"last":3.0939999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.5760000000000001,"15min":1.2030000000000001},"min":{"1min":0.41999999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.609,"5min":6.609,"15min":6.609},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.415,"5min":1.873,"15min":1.325},"min":{"1min":0.55300000000000005,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.8070000000000004,"5min":11.754,"15min":11.754},"last":0.78500000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.1439999999999999,"15min":1.1140000000000001},"min":{"1min":0.36899999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.2370000000000001,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":1.3},{"interface":"front","average":{"1min":1.163,"5min":1.4199999999999999,"15min":1.1879999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.633,"5min":6.0250000000000004,"15min":6.0250000000000004},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:23:55 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.5609999999999999,"15min":1.222},"min":{"1min":0.47599999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.391,"5min":6.4820000000000002,"15min":6.4820000000000002},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.5649999999999999,"15min":1.268},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.391,"5min":11.439,"15min":11.439},"last":3.1400000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019212,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1838148,"kb_used_data":1528636,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91907004,"statfs":{"total":95995035648,"available":94112772096,"internally_reserved":0,"allocated":1565323264,"data_stored":1564804465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.2029999999999998,"15min":1.861},"min":{"1min":0.28999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5780000000000003,"5min":84.888000000000005,"15min":84.888000000000005},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.5150000000000001,"15min":1.9930000000000001},"min":{"1min":0.41299999999999998,"5min":0.312,"15min":0.312},"max":{"1min":4.6310000000000002,"5min":84.912999999999997,"15min":84.912999999999997},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":2.3300000000000001,"15min":1.4430000000000001},"min":{"1min":0.41399999999999998,"5min":0.28199999999999997,"15min":0.23999999999999999},"max":{"1min":4.9290000000000003,"5min":61.726999999999997,"15min":61.726999999999997},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2030000000000001,"5min":2.9700000000000002,"15min":1.6910000000000001},"min":{"1min":0.34599999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.944,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.44700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":3.2719999999999998,"15min":1.718},"min":{"1min":0.35499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.103,"5min":61.783999999999999,"15min":61.783999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.4860000000000002,"15min":1.784},"min":{"1min":0.53100000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.2909999999999999,"5min":61.551000000000002,"15min":61.551000000000002},"last":2.597}]},{"osd":3,"last update":"Mon Oct 9 06:23:59 2023","interfaces":[{"interface":"back","average":{"1min":5.0519999999999996,"5min":2.7599999999999998,"15min":1.716},"min":{"1min":0.36599999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":91.361999999999995,"5min":91.361999999999995,"15min":91.361999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":4.9249999999999998,"5min":2.1429999999999998,"15min":1.5129999999999999},"min":{"1min":0.36699999999999999,"5min":0.313,"15min":0.313},"max":{"1min":89.676000000000002,"5min":89.676000000000002,"15min":89.676000000000002},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:23:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.8109999999999999,"15min":1.2230000000000001},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.0030000000000001,"5min":12.242000000000001,"15min":12.242000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.77100000000000002,"5min":2.5129999999999999,"15min":1.4930000000000001},"min":{"1min":0.34100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.2470000000000001,"5min":20.113,"15min":20.113},"last":0.96199999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:23:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.1680000000000001,"15min":1.4219999999999999},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.1150000000000002,"5min":18.010000000000002,"15min":18.010000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.6439999999999999,"15min":1.24},"min":{"1min":0.48599999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.679,"5min":13.913,"15min":13.913},"last":0.42899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.306,"15min":1.496},"min":{"1min":0.58399999999999996,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.452,"5min":15.294,"15min":15.294},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.012,"5min":2.3380000000000001,"15min":1.504},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.575,"5min":10.731999999999999,"15min":10.731999999999999},"last":0.84299999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822979,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1704176,"kb_used_data":1406952,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92040976,"statfs":{"total":95995035648,"available":94249959424,"internally_reserved":0,"allocated":1440718848,"data_stored":1440215662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":3.3849999999999998,"15min":1.804},"min":{"1min":0.311,"5min":0.247,"15min":0.247},"max":{"1min":1.1020000000000001,"5min":69.793999999999997,"15min":69.793999999999997},"last":2.3780000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":2.847,"15min":1.5960000000000001},"min":{"1min":0.375,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.2589999999999999,"5min":68.873000000000005,"15min":68.873000000000005},"last":1.6879999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.5619999999999998,"15min":1.476},"min":{"1min":0.372,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3409999999999993,"5min":52.070999999999998,"15min":52.070999999999998},"last":0.309},{"interface":"front","average":{"1min":1.363,"5min":2.8570000000000002,"15min":1.5580000000000001},"min":{"1min":0.48999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.8700000000000001,"5min":51.570999999999998,"15min":51.570999999999998},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":3.109,"15min":1.6519999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.307},"max":{"1min":1.1859999999999999,"5min":47.207000000000001,"15min":47.207000000000001},"last":2.7250000000000001},{"interface":"front","average":{"1min":0.875,"5min":3.0550000000000002,"15min":1.718},"min":{"1min":0.46600000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.347,"5min":59.082999999999998,"15min":59.082999999999998},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:24:00 2023","interfaces":[{"interface":"back","average":{"1min":5.7789999999999999,"5min":2.778,"15min":1.6100000000000001},"min":{"1min":0.42699999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":66.153999999999996,"5min":66.153999999999996,"15min":66.153999999999996},"last":2.6840000000000002},{"interface":"front","average":{"1min":6.085,"5min":2.641,"15min":1.575},"min":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":65.792000000000002,"5min":65.792000000000002,"15min":65.792000000000002},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:23:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":2.6709999999999998,"15min":1.494},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.669,"5min":17.667999999999999,"15min":17.667999999999999},"last":2.4980000000000002},{"interface":"front","average":{"1min":1.468,"5min":2.202,"15min":1.4119999999999999},"min":{"1min":0.52400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.9720000000000004,"5min":16.181999999999999,"15min":16.181999999999999},"last":2.4159999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":2.1560000000000001,"15min":1.377},"min":{"1min":0.41799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.6669999999999998,"5min":17.454999999999998,"15min":17.454999999999998},"last":1.6719999999999999},{"interface":"front","average":{"1min":1.01,"5min":2.0710000000000002,"15min":1.4279999999999999},"min":{"1min":0.34899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.069,"5min":12.365,"15min":12.365},"last":2.4500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":2.339,"15min":1.532},"min":{"1min":0.64300000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.5740000000000001,"5min":13.047000000000001,"15min":13.047000000000001},"last":2.7599999999999998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":2.1360000000000001,"15min":1.48},"min":{"1min":0.438,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.679,"5min":11.981,"15min":11.981},"last":0.40899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1122959360,"data_stored":1122959360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1493041152,"data_stored":1493041152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2372534272,"data_stored":2372534272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2936406016,"data_stored":2936406016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1435238400,"data_stored":1435238400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1440940032,"data_stored":1440940032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1440088064,"data_stored":1440088064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1720647680,"data_stored":1720647680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:24:15.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-09T06:24:15.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:15.992 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:15 smithi125 bash[77299]: cluster 2023-10-09T06:24:14.902530+0000 mgr.y (mgr.25723) 574 : cluster 0 pgmap v545: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:16.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:16.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:16.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:15 smithi155 bash[59495]: cluster 2023-10-09T06:24:14.902530+0000 mgr.y (mgr.25723) 574 : cluster 0 pgmap v545: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:16.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 76851 76835 92.3323 64 0.0326339 0.0108235 2023-10-09T06:24:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:16.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:16.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:16.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:16.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:15 smithi125 bash[80002]: cluster 2023-10-09T06:24:14.902530+0000 mgr.y (mgr.25723) 574 : cluster 0 pgmap v545: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:16.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:16.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:16.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:16.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:16.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:16.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:16.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:16.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:16.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:16.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:17.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:17.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:17.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 77889 77873 91.8142 64.875 0.0111501 0.0108863 2023-10-09T06:24:17.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:17.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:17.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:17.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:17.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:17.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:17.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:17.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:17.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:17.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:17.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:17.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:17.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:17.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:17.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:17.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:24:17.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:24:18.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:18.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 78859 78843 91.2364 60.625 0.0225373 0.0109539 2023-10-09T06:24:18.182 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:17 smithi155 bash[59495]: cluster 2023-10-09T06:24:16.904234+0000 mgr.y (mgr.25723) 575 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:24:18.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:18.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:18.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:18.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:17 smithi125 bash[77299]: cluster 2023-10-09T06:24:16.904234+0000 mgr.y (mgr.25723) 575 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:24:18.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:17 smithi125 bash[80002]: cluster 2023-10-09T06:24:16.904234+0000 mgr.y (mgr.25723) 575 : cluster 0 pgmap v546: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:24:18.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:18.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:18.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:18.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:18.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:18.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:18.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:18.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:18.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:18.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:19.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:19.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:19.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:19.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:18 smithi125 bash[77299]: audit 2023-10-09T06:24:18.572448+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:19.125 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:18 smithi125 bash[80002]: audit 2023-10-09T06:24:18.572448+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:19.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:19.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:19.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 79827 79811 90.6775 60.5 0.0263769 0.0110228 2023-10-09T06:24:19.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:19.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:18 smithi155 bash[59495]: audit 2023-10-09T06:24:18.572448+0000 mon.a (mon.0) 785 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:19.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:19.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:19.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:19.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:19.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:19.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:19.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:19.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:20.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:20.308 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 80846 80830 90.1954 63.6875 0.0273363 0.0110816 2023-10-09T06:24:20.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:20 smithi155 bash[59495]: cluster 2023-10-09T06:24:18.904726+0000 mgr.y (mgr.25723) 576 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:20.354 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:19 smithi125 bash[77299]: cluster 2023-10-09T06:24:18.904726+0000 mgr.y (mgr.25723) 576 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:20.690 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:19 smithi125 bash[80002]: cluster 2023-10-09T06:24:18.904726+0000 mgr.y (mgr.25723) 576 : cluster 0 pgmap v547: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:24:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:20.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:20.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:20.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:21.078 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:21.078 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:21.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 81841 81825 89.704 62.1875 0.0164696 0.0111423 2023-10-09T06:24:21.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:21.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:21.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:21.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:21.550 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:21] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:24:21.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:21.671 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:21.714 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:21.814 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:21.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:21.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:21.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:21.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:21.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:22.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:22.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:22.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:22 smithi125 bash[77299]: cluster 2023-10-09T06:24:20.905486+0000 mgr.y (mgr.25723) 577 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T06:24:22.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:22 smithi125 bash[80002]: cluster 2023-10-09T06:24:20.905486+0000 mgr.y (mgr.25723) 577 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T06:24:22.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 82812 82796 89.2037 60.6875 0.0156687 0.0112039 2023-10-09T06:24:22.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.250 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:22.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:22.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:22.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:22.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:22 smithi155 bash[59495]: cluster 2023-10-09T06:24:20.905486+0000 mgr.y (mgr.25723) 577 : cluster 0 pgmap v548: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-10-09T06:24:22.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:22.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:22.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:22.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:22.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:22.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:22.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:22.836 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:22.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:22.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:22.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:23.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:23.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:23.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:23.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 83835 83819 88.7754 63.9375 0.0235846 0.011259 2023-10-09T06:24:23.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:23.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:23.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:23.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:23.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:23.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:23.550 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:23.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:23.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:23.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:23.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:23.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:23.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:23.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:23.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:23.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:24.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:24.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:24:24.169504+0000 min lat: 0.00129902 max lat: 0.219491 avg lat: 0.0113118 2023-10-09T06:24:24.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:24:24.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 84857 84841 88.3603 63.875 0.0147016 0.0113118 2023-10-09T06:24:24.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:24.228 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:24 smithi155 bash[59495]: cluster 2023-10-09T06:24:22.906074+0000 mgr.y (mgr.25723) 578 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T06:24:24.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:24.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:24 smithi125 bash[77299]: cluster 2023-10-09T06:24:22.906074+0000 mgr.y (mgr.25723) 578 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T06:24:24.358 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:24 smithi125 bash[80002]: cluster 2023-10-09T06:24:22.906074+0000 mgr.y (mgr.25723) 578 : cluster 0 pgmap v549: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-09T06:24:24.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:24.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:24.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:24.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:24.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:24.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:24.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:24.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:24.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:25.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:25.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:25.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:25.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 85864 85848 87.9408 62.9375 0.0162715 0.0113647 2023-10-09T06:24:25.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:25.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:25.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:25.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:25.475 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:25.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:25.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:25.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:25.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:25.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:25.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:25.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:25.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:26.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:26.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:26 smithi125 bash[77299]: cluster 2023-10-09T06:24:24.906660+0000 mgr.y (mgr.25723) 579 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:24:26.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:26 smithi125 bash[80002]: cluster 2023-10-09T06:24:24.906660+0000 mgr.y (mgr.25723) 579 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:24:26.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:26.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 86876 86860 87.5425 63.25 0.0101661 0.0114173 2023-10-09T06:24:26.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:26.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:26.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:26 smithi155 bash[59495]: cluster 2023-10-09T06:24:24.906660+0000 mgr.y (mgr.25723) 579 : cluster 0 pgmap v550: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:24:26.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:26.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:26.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:26.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:26.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:26.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:26.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:26.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:27.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:27.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:27.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:27.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 87785 87769 87.0547 56.8125 0.0386939 0.0114807 2023-10-09T06:24:27.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:27.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:27.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:27.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:27.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:27.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:27.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:27.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:27.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:27.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:27.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:27.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:27.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:27.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:28.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:28.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:28.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:28 smithi155 bash[59495]: cluster 2023-10-09T06:24:26.908015+0000 mgr.y (mgr.25723) 580 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:24:28.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 88641 88625 86.5304 53.5 0.00423321 0.0115371 2023-10-09T06:24:28.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:28.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:28.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:28.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:28 smithi125 bash[80002]: cluster 2023-10-09T06:24:26.908015+0000 mgr.y (mgr.25723) 580 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:24:28.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:28 smithi125 bash[77299]: cluster 2023-10-09T06:24:26.908015+0000 mgr.y (mgr.25723) 580 : cluster 0 pgmap v551: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:24:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:28.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:28.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:28.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:28.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:28.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:28.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:28.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:29.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:29.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:29.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:29.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 89625 89609 86.1416 61.5 0.0137194 0.0116022 2023-10-09T06:24:29.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:29.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:29.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:29.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:29.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:29.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:29.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:29 smithi155 bash[59495]: cluster 2023-10-09T06:24:28.908437+0000 mgr.y (mgr.25723) 581 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:29.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:29.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:29.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:29 smithi125 bash[77299]: cluster 2023-10-09T06:24:28.908437+0000 mgr.y (mgr.25723) 581 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:29 smithi125 bash[80002]: cluster 2023-10-09T06:24:28.908437+0000 mgr.y (mgr.25723) 581 : cluster 0 pgmap v552: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:29.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:29.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:29.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:30.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:30.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:30.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 90609 90593 85.7682 61.5 0.0170465 0.0116542 2023-10-09T06:24:30.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:30.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:30.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:30.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:30.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:30.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:30.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:30.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:30.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:30.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:30.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:30.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:30.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:31.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:31.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:31.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 91583 91567 85.3966 60.875 0.00798005 0.0117051 2023-10-09T06:24:31.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:31.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:31.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:31.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:31.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:31.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:31.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:31.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:31.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:31] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:24:31.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:31.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:31.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:31.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:31.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:31.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:31.994 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:31 smithi125 bash[80002]: cluster 2023-10-09T06:24:30.909022+0000 mgr.y (mgr.25723) 582 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:24:31.994 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:31 smithi125 bash[77299]: cluster 2023-10-09T06:24:30.909022+0000 mgr.y (mgr.25723) 582 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:32.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:31 smithi155 bash[59495]: cluster 2023-10-09T06:24:30.909022+0000 mgr.y (mgr.25723) 582 : cluster 0 pgmap v553: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:24:32.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:32.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:32.179 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 92545 92529 85.0249 60.125 0.0158626 0.0117558 2023-10-09T06:24:32.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:32.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:32.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:32.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:32.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:32.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:32.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:32.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:32.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:32.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:32.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:32.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:32.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:33.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:33.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:33.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:33.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 93470 93454 84.6305 57.8125 0.00479414 0.0118092 2023-10-09T06:24:33.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:33.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:33.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:33.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:33.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:33.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:33.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:33.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:33.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:33.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:33.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:33.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:33.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:33.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:33.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:34.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:34.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:34.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:33 smithi125 bash[77299]: cluster 2023-10-09T06:24:32.909611+0000 mgr.y (mgr.25723) 583 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:24:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:33 smithi125 bash[77299]: audit 2023-10-09T06:24:33.572779+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:34.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:33 smithi125 bash[80002]: cluster 2023-10-09T06:24:32.909611+0000 mgr.y (mgr.25723) 583 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:24:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:33 smithi125 bash[80002]: audit 2023-10-09T06:24:33.572779+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:34.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:34.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 94341 94325 84.1992 54.4375 0.0127854 0.0118707 2023-10-09T06:24:34.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:34.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:34.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:34.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:34.388 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:33 smithi155 bash[59495]: cluster 2023-10-09T06:24:32.909611+0000 mgr.y (mgr.25723) 583 : cluster 0 pgmap v554: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:24:34.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:33 smithi155 bash[59495]: audit 2023-10-09T06:24:33.572779+0000 mon.a (mon.0) 786 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:34.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:34.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:34.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:34.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:34.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:34.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:34.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:34.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:34.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:34.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:35.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:35.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:35.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 95263 95247 83.8249 57.625 0.00728683 0.0119237 2023-10-09T06:24:35.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:35.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:35.364 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:35.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:35.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:35.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:35.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:35.670 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:35.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:35.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:35.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:35.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:35.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:36.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:36.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:36.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:35 smithi155 bash[59495]: cluster 2023-10-09T06:24:34.910130+0000 mgr.y (mgr.25723) 584 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:36.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 96153 96137 83.4332 55.625 0.0298455 0.0119802 2023-10-09T06:24:36.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.198 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:35 smithi125 bash[77299]: cluster 2023-10-09T06:24:34.910130+0000 mgr.y (mgr.25723) 584 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:36.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:35 smithi125 bash[80002]: cluster 2023-10-09T06:24:34.910130+0000 mgr.y (mgr.25723) 584 : cluster 0 pgmap v555: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:24:36.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:36.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:24:36.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:36.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:36.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:36.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:36.622 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76676 5.4 5:281b0f99:::benchmark_data_smithi125_106927_object76675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:59.790311+0000", 2023-10-09T06:24:36.700 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 336.90041984099997, 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.366389624, 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76676 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790311+0000", 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790311+0000", 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790309+0000", 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:36.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790431+0000", 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012209699999999999 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790445+0000", 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4158000000000001e-05 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790457+0000", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1527e-05 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790531+0000", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4043999999999998e-05 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790544+0000", 2023-10-09T06:24:36.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2771e-05 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.112277+0000", 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3217328669999999 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.114499+0000", 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002222108 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.114708+0000", 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000209323 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.126430+0000", 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011721519 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.126481+0000", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1094999999999998e-05 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136630+0000", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010148935 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.155228+0000", 2023-10-09T06:24:36.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018598017000000001 2023-10-09T06:24:36.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.155290+0000", 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2150000000000006e-05 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.156701+0000", 2023-10-09T06:24:36.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001410962 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76684 5.4 5:22f6ecc8:::benchmark_data_smithi125_106927_object76683:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:59.790478+0000", 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 336.90025255099999, 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3676465339999999, 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.710 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76684 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790478+0000", 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790478+0000", 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790478+0000", 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790545+0000", 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7183999999999999e-05 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790556+0000", 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1686e-05 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790565+0000", 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7930000000000003e-06 2023-10-09T06:24:36.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790797+0000", 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023143799999999999 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790803+0000", 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2729999999999999e-06 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.123327+0000", 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.332524005 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.123482+0000", 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015500400000000001 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:24:36.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.123656+0000", 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017357200000000001 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136637+0000", 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012981357000000001 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136686+0000", 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8721999999999999e-05 2023-10-09T06:24:36.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136797+0000", 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111758 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.156765+0000", 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019968054999999998 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.156806+0000", 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0645000000000001e-05 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158125+0000", 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013188119999999999 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76687 5.4 5:2cae2000:::benchmark_data_smithi125_106927_object76686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e118)", 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:59.790590+0000", 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 336.90014141299997, 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3737791320000001, 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76687 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.719 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790590+0000", 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790590+0000", 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790589+0000", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790750+0000", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016117 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790759+0000", 2023-10-09T06:24:36.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9760000000000001e-06 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790764+0000", 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3519999999999997e-06 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790816+0000", 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1248999999999997e-05 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.790819+0000", 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.721e-06 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.126579+0000", 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.335759938 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.126773+0000", 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000193747 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.126979+0000", 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020558800000000001 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136695+0000", 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0097164710000000008 2023-10-09T06:24:36.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136743+0000", 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8430000000000002e-05 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.136833+0000", 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9581999999999997e-05 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158177+0000", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021343562999999999 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158327+0000", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014988100000000001 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.164369+0000", 2023-10-09T06:24:36.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060421659999999999 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76743 5.5 5:aee4eb29:::benchmark_data_smithi125_106927_object76742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.172293+0000", 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 335.51843840100003, 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.204102638, 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76743 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172293+0000", 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172293+0000", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172291+0000", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172327+0000", 2023-10-09T06:24:36.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6078000000000003e-05 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172345+0000", 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7552999999999999e-05 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172354+0000", 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.833e-06 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172412+0000", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8705999999999997e-05 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172596+0000", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000183296 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172763+0000", 2023-10-09T06:24:36.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016730099999999999 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.178005+0000", 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0052418949999999999 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.181148+0000", 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031429460000000002 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376305+0000", 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.195157833 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376355+0000", 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9252000000000002e-05 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376395+0000", 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0442999999999999e-05 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76744 5.5 5:a303caca:::benchmark_data_smithi125_106927_object76743:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.172378+0000", 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 335.51835259699999, 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20419060999999999, 2023-10-09T06:24:36.734 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76744 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172378+0000", 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172378+0000", 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172378+0000", 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172401+0000", 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3329e-05 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172410+0000", 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3589999999999997e-06 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.172417+0000", 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3929999999999996e-06 2023-10-09T06:24:36.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.175407+0000", 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029897320000000002 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.175532+0000", 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012513800000000001 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.175693+0000", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016181799999999999 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.178296+0000", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026020930000000002 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.181220+0000", 2023-10-09T06:24:36.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002924149 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376458+0000", 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.195238199 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376509+0000", 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1402e-05 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376569+0000", 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9648999999999998e-05 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76759 5.5 5:a38dba55:::benchmark_data_smithi125_106927_object76758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.191029+0000", 2023-10-09T06:24:36.741 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 335.49970201399998, 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18566174599999999, 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76759 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191029+0000", 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191029+0000", 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191027+0000", 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:36.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191050+0000", 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2804999999999999e-05 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191106+0000", 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5937000000000003e-05 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191114+0000", 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.388e-06 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191174+0000", 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9862999999999999e-05 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191361+0000", 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018656200000000001 2023-10-09T06:24:36.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.191561+0000", 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019998699999999999 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.196425+0000", 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0048649050000000001 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.201090+0000", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046646120000000003 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376617+0000", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17552720999999999 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376656+0000", 2023-10-09T06:24:36.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8918999999999998e-05 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.376691+0000", 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4489999999999997e-05 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76760 5.5 5:aa2e1611:::benchmark_data_smithi125_106927_object76759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:24:36.748 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.193043+0000", 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 335.49768806499998, 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18602965499999999, 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76760 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193043+0000", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193043+0000", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193042+0000", 2023-10-09T06:24:36.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193068+0000", 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6568e-05 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193080+0000", 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1769e-05 2023-10-09T06:24:36.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.193087+0000", 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5150000000000003e-06 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.194254+0000", 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001167967 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.194414+0000", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015938000000000001 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.194571+0000", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015743500000000001 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.196586+0000", 2023-10-09T06:24:36.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020144640000000001 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.201148+0000", 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045621710000000003 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.378938+0000", 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17779044299999999 2023-10-09T06:24:36.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.378997+0000", 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8792999999999998e-05 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.379073+0000", 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5372999999999998e-05 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:85680 5.4 5:2e7ceb2a:::benchmark_data_smithi125_106927_object85679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:12.475891+0000", 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 324.21484024699998, 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15639793399999999, 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.756 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 85680 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475891+0000", 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475891+0000", 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475889+0000", 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475913+0000", 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3277000000000001e-05 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475922+0000", 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5419999999999994e-06 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475929+0000", 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2820000000000002e-06 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.475976+0000", 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7688000000000001e-05 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.506668+0000", 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.030691970999999998 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.506856+0000", 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000187999 2023-10-09T06:24:36.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.509989+0000", 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031324590000000002 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.510804+0000", 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00081538300000000004 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.514289+0000", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003485049 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.629170+0000", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.114880526 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.629286+0000", 2023-10-09T06:24:36.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116088 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:12.632289+0000", 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030029150000000001 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.763 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:86971 5.4 5:26c89492:::benchmark_data_smithi125_106927_object86970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.480592+0000", 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.21013850600002, 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.193319201, 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 86971 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480592+0000", 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480592+0000", 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480591+0000", 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480641+0000", 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0386000000000002e-05 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480654+0000", 2023-10-09T06:24:36.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2727e-05 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480663+0000", 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5820000000000002e-06 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480736+0000", 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3813000000000006e-05 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.480935+0000", 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019904100000000001 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5,7", 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.481232+0000", 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029699400000000002 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.485593+0000", 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043608070000000004 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488593+0000", 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030000309999999998 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.490519+0000", 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019261510000000001 2023-10-09T06:24:36.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671964+0000", 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18144487100000001 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672073+0000", 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010925999999999999 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.673912+0000", 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00183815 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:86974 5.c e121/105 5:370d6580:::benchmark_data_smithi125_106927_object86973:head v 121'5440, mlcod=121'5437)", 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.488642+0000", 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.20208910399998, 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18330545200000001, 2023-10-09T06:24:36.771 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488642+0000", 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488642+0000", 2023-10-09T06:24:36.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488641+0000", 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488662+0000", 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0494000000000001e-05 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488672+0000", 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0601999999999999e-05 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488679+0000", 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5529999999999998e-06 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488709+0000", 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0394000000000001e-05 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.488728+0000", 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8493000000000001e-05 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671856+0000", 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18312868900000001 2023-10-09T06:24:36.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671947+0000", 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0854999999999997e-05 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:86972 5.2 e121/105 5:4da22672:::benchmark_data_smithi125_106927_object86971:head v 121'5396, mlcod=121'5394)", 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.489612+0000", 2023-10-09T06:24:36.776 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.20111940800001, 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20217953999999999, 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489612+0000", 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489612+0000", 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489611+0000", 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489645+0000", 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3872999999999999e-05 2023-10-09T06:24:36.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489655+0000", 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0829000000000001e-05 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489662+0000", 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4699999999999999e-06 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489730+0000", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7741000000000001e-05 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.489748+0000", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8343e-05 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.691760+0000", 2023-10-09T06:24:36.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.202012044 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.691791+0000", 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1093999999999997e-05 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.781 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:86978 5.5 5:a10be9d4:::benchmark_data_smithi125_106927_object86977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.493023+0000", 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.197708014, 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.18639851099999999, 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 86978 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493023+0000", 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493023+0000", 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493021+0000", 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493057+0000", 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5305000000000002e-05 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493070+0000", 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3760999999999999e-05 2023-10-09T06:24:36.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493080+0000", 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7210000000000006e-06 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493177+0000", 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6751999999999998e-05 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671904+0000", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17872722199999999 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672077+0000", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000173158 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.678424+0000", 2023-10-09T06:24:36.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0063467790000000003 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.678433+0000", 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7080000000000003e-06 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.679175+0000", 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00074222399999999999 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.679315+0000", 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014033700000000001 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.679421+0000", 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106131 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:86977 5.2 e121/105 5:423bd8b3:::benchmark_data_smithi125_106927_object86976:head v 121'5397, mlcod=121'5394)", 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.493859+0000", 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.19687240399998, 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.197962793, 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493859+0000", 2023-10-09T06:24:36.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493859+0000", 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493858+0000", 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:36.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493987+0000", 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012907399999999999 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671761+0000", 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17777379800000001 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671782+0000", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1177000000000001e-05 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672484+0000", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070175900000000004 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672584+0000", 2023-10-09T06:24:36.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010013099999999999 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.691798+0000", 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.019214663999999999 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.691821+0000", 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2853e-05 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.793 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44693.0:86971 5.4 e121/118 ondisk, result = 0)", 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.493863+0000", 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.196867962, 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.180056888, 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493863+0000", 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493863+0000", 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493862+0000", 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.493868+0000", 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.49e-06 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671789+0000", 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17792134000000001 2023-10-09T06:24:36.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671814+0000", 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5072999999999999e-05 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671914+0000", 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9858000000000005e-05 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671943+0000", 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9246000000000001e-05 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.673920+0000", 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001976654 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44693.0:86987 5.c e121/105 5:37fa2b22:::benchmark_data_smithi125_106927_object86986:head v 121'5441, mlcod=121'5438)", 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:14.507530+0000", 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 322.18320068600002, 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.179834146, 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.507530+0000", 2023-10-09T06:24:36.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.507530+0000", 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.507529+0000", 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.507559+0000", 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0165e-05 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671821+0000", 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.16426178399999999 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.671843+0000", 2023-10-09T06:24:36.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1985999999999999e-05 2023-10-09T06:24:36.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672026+0000", 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018354999999999999 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.672082+0000", 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6064999999999999e-05 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.687332+0000", 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015249845 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:14.687364+0000", 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2066000000000001e-05 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:33130 6.0 6:06c17677:::benchmark_data_smithi125_130224_object33129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:39.611306+0000", 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 57.079425327000003, 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17140286299999999, 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 33130 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.806 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611306+0000", 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611306+0000", 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.808 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611304+0000", 2023-10-09T06:24:36.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611348+0000", 2023-10-09T06:24:36.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3847999999999998e-05 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611364+0000", 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5965e-05 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611378+0000", 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3406e-05 2023-10-09T06:24:36.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.621749+0000", 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010371516000000001 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.621861+0000", 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111606 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.621999+0000", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000138613 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628275+0000", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.006275465 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628320+0000", 2023-10-09T06:24:36.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4858000000000003e-05 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.782521+0000", 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.154201582 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.782646+0000", 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012465800000000001 2023-10-09T06:24:36.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.782708+0000", 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2500000000000001e-05 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:33131 6.0 6:03258495:::benchmark_data_smithi125_130224_object33130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:39.611390+0000", 2023-10-09T06:24:36.814 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 57.079340619, 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17366290400000001, 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 33131 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.815 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611390+0000", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611390+0000", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611389+0000", 2023-10-09T06:24:36.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611434+0000", 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4493999999999998e-05 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611450+0000", 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6523000000000001e-05 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611464+0000", 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4134000000000001e-05 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628765+0000", 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017300426000000001 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628910+0000", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014486999999999999 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628995+0000", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4915999999999998e-05 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633268+0000", 2023-10-09T06:24:36.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042735220000000001 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633391+0000", 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012275900000000001 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.784784+0000", 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15139356700000001 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785001+0000", 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021624799999999999 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785053+0000", 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2543000000000002e-05 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:33137 6.0 6:0e07741c:::benchmark_data_smithi125_130224_object33136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:39.611566+0000", 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 57.079165119000002, 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17363638200000001, 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:24:36.822 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 33137 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611566+0000", 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611566+0000", 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.823 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611565+0000", 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611609+0000", 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4292999999999998e-05 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611628+0000", 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8754e-05 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.611646+0000", 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8637000000000001e-05 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.630272+0000", 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.018626045000000001 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.630395+0000", 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012227399999999999 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.630728+0000", 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033347200000000002 2023-10-09T06:24:36.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633274+0000", 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002545472 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633599+0000", 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00032565200000000001 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785120+0000", 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15152023000000001 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785164+0000", 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4793000000000003e-05 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.828 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785202+0000", 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7914000000000003e-05 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:33159 6.0 6:08762a0c:::benchmark_data_smithi125_130224_object33158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:39.628352+0000", 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 57.062378985999999, 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15699944900000001, 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.829 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 33159 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628352+0000", 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628352+0000", 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628351+0000", 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628377+0000", 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6492e-05 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628386+0000", 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8720000000000008e-06 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.628392+0000", 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2739999999999998e-06 2023-10-09T06:24:36.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.631913+0000", 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035214930000000001 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.632047+0000", 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134349 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.833 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.632182+0000", 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000134547 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633488+0000", 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013059110000000001 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.633824+0000", 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033573799999999998 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785250+0000", 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.151426386 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785319+0000", 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8725000000000004e-05 2023-10-09T06:24:36.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:39.785351+0000", 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2663999999999998e-05 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:41669 6.9 6:9e09cb33:::benchmark_data_smithi125_130224_object41668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:45.726787+0000", 2023-10-09T06:24:36.836 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 50.963944228000003, 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.149661238, 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 41669 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726787+0000", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726787+0000", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726786+0000", 2023-10-09T06:24:36.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726810+0000", 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3788999999999999e-05 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726818+0000", 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5129999999999996e-06 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726826+0000", 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0890000000000002e-06 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726873+0000", 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7899000000000003e-05 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.840 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.726991+0000", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011769699999999999 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.727114+0000", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122748 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.728954+0000", 2023-10-09T06:24:36.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001840129 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.729716+0000", 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000761946 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.876367+0000", 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.14665120000000001 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.876413+0000", 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6124000000000001e-05 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:45.876448+0000", 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4674e-05 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.843 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:24:36.844 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:24:36.844 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:24:36.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:36.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:36.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:36.904 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:24:36.947 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:24:36.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.991 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:24:37.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:37.048 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:24:37.079 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:24:37.107 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:24:37.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:37.118 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[77299]: audit 2023-10-09T06:24:36.287302+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.118 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[77299]: audit 2023-10-09T06:24:36.290744+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.125:0/3073093317' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.118 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[77299]: audit 2023-10-09T06:24:36.853721+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.125:0/170370410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:37.119 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[80002]: audit 2023-10-09T06:24:36.287302+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.119 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[80002]: audit 2023-10-09T06:24:36.290744+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.125:0/3073093317' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.119 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:37 smithi125 bash[80002]: audit 2023-10-09T06:24:36.853721+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.125:0/170370410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:37.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:37.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:37.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:36 smithi155 bash[59495]: audit 2023-10-09T06:24:36.287302+0000 mon.a (mon.0) 787 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:36 smithi155 bash[59495]: audit 2023-10-09T06:24:36.290744+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.125:0/3073093317' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:24:37.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:36 smithi155 bash[59495]: audit 2023-10-09T06:24:36.853721+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.125:0/170370410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:37.143 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:24:37.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 96919 96903 82.9461 47.875 0.025168 0.0120496 2023-10-09T06:24:37.182 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.191 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:24:37.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:37.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:37.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:37.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:37.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:37.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:37.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:37.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:37.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:37.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:37.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:37.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:37.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:37.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:38.027 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:24:38.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:38.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[77299]: cluster 2023-10-09T06:24:36.910672+0000 mgr.y (mgr.25723) 585 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T06:24:38.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[77299]: audit 2023-10-09T06:24:37.394941+0000 mon.a (mon.0) 788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[77299]: audit 2023-10-09T06:24:37.399001+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.125:0/2922740952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[80002]: cluster 2023-10-09T06:24:36.910672+0000 mgr.y (mgr.25723) 585 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T06:24:38.055 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[80002]: audit 2023-10-09T06:24:37.394941+0000 mon.a (mon.0) 788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.056 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:38 smithi125 bash[80002]: audit 2023-10-09T06:24:37.399001+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.125:0/2922740952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:38.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:38.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 97775 97759 82.5482 53.5 0.00482294 0.0121046 2023-10-09T06:24:38.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:38.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:38.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:38 smithi155 bash[59495]: cluster 2023-10-09T06:24:36.910672+0000 mgr.y (mgr.25723) 585 : cluster 0 pgmap v556: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T06:24:38.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:38 smithi155 bash[59495]: audit 2023-10-09T06:24:37.394941+0000 mon.a (mon.0) 788 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:38 smithi155 bash[59495]: audit 2023-10-09T06:24:37.399001+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.125:0/2922740952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:38.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:38.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:38.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:38.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:38.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:38.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:38.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:38.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:38.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:38.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:38.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:39.078 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:24:39.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:39.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 98527 98511 82.0742 47 0.0204689 0.0121783 2023-10-09T06:24:39.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:37.992488+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:39.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: audit 2023-10-09T06:24:37.999876+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:39.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:37.999964+0000 mon.a (mon.0) 791 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.013646+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.014332+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.016397+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.018025+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.024345+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:37.992488+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: audit 2023-10-09T06:24:37.999876+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:37.999964+0000 mon.a (mon.0) 791 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.013646+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.014332+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.016397+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.018025+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:39 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.024345+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:39.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.361 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:37.992488+0000 mon.a (mon.0) 789 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: audit 2023-10-09T06:24:37.999876+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:37.999964+0000 mon.a (mon.0) 791 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.013646+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.014332+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.016397+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.018025+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.362 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:39 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.024345+0000 osd.4 (osd.4) 76 : cluster 3 failed to encode map e149 with expected crc 2023-10-09T06:24:39.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:39.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:39.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:39.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:39.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:39.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:39.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:39.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:39.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:39.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:40.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:40.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:40.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:40 smithi125 bash[77299]: cluster 2023-10-09T06:24:38.911285+0000 mgr.y (mgr.25723) 586 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:24:40.069 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:40 smithi125 bash[80002]: cluster 2023-10-09T06:24:38.911285+0000 mgr.y (mgr.25723) 586 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:24:40.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:40.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:40.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 99474 99458 81.773 59.1875 0.0234975 0.0122235 2023-10-09T06:24:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:40.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:40.291 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:40 smithi155 bash[59495]: cluster 2023-10-09T06:24:38.911285+0000 mgr.y (mgr.25723) 586 : cluster 0 pgmap v558: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-10-09T06:24:40.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:40.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:40.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:40.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:40.548 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:40.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:40.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:40.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:40.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:40.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:40.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:40.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:40.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:41.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:24:41.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:41.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:41.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 100375 100359 81.4424 56.3125 0.0234692 0.0122718 2023-10-09T06:24:41.215 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:41] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:24:41.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:41.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:41.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:41.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:41.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:41.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:41.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:41.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:41.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:41.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:41.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:41.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:41.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:42.023 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:42 smithi125 bash[77299]: cluster 2023-10-09T06:24:40.912201+0000 mgr.y (mgr.25723) 587 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:24:42.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:42.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.180 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 101261 101245 81.1081 55.375 0.025691 0.0123236 2023-10-09T06:24:42.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:42.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:42 smithi125 bash[80002]: cluster 2023-10-09T06:24:40.912201+0000 mgr.y (mgr.25723) 587 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:24:42.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:42.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:42 smithi155 bash[59495]: cluster 2023-10-09T06:24:40.912201+0000 mgr.y (mgr.25723) 587 : cluster 0 pgmap v559: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:24:42.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:42.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:42.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:42.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:42.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:42.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:42.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:42.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:42.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:42.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:43.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:43.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:43.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:43.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:24:43.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:24:43.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 102107 102091 80.7498 52.875 0.0186136 0.0123781 2023-10-09T06:24:43.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:43.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:43.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:43.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:43.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:43.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:43.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:43.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:43.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:43.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:43.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:43.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:43.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:44.080 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:24:44.080 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:24:44.080 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-09T06:24:44.081 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:24:44.081 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:24:44.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-09T06:24:44.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:44.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:24:44.176854+0000 min lat: 0.00129902 max lat: 0.219491 avg lat: 0.0124389 2023-10-09T06:24:44.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:24:44.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 102893 102877 80.3545 49.125 0.0146323 0.0124389 2023-10-09T06:24:44.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:44.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:44.369 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:44 smithi125 bash[80002]: cluster 2023-10-09T06:24:42.912825+0000 mgr.y (mgr.25723) 588 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:44.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:44 smithi125 bash[77299]: cluster 2023-10-09T06:24:42.912825+0000 mgr.y (mgr.25723) 588 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:44.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:44 smithi155 bash[59495]: cluster 2023-10-09T06:24:42.912825+0000 mgr.y (mgr.25723) 588 : cluster 0 pgmap v560: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-09T06:24:44.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:44.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:44.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:44.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:44.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:44.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:44.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:44.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:44.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:44.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:45.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:45.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:45.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:45.181 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 103722 103706 80.0021 51.8125 0.00952731 0.0124938 2023-10-09T06:24:45.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:45.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:45.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:45.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:45.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:45.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:45.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:45.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:45.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:45.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:45.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:45.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:45.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:45.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:45.888 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:45.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:45.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:46.048 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:46 smithi125 bash[77299]: cluster 2023-10-09T06:24:44.913526+0000 mgr.y (mgr.25723) 589 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:24:46.048 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:46 smithi125 bash[80002]: cluster 2023-10-09T06:24:44.913526+0000 mgr.y (mgr.25723) 589 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:24:46.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:46.112 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:46 smithi155 bash[59495]: cluster 2023-10-09T06:24:44.913526+0000 mgr.y (mgr.25723) 589 : cluster 0 pgmap v561: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:24:46.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:46.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 104552 104536 79.6591 51.875 0.0116317 0.0125472 2023-10-09T06:24:46.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:46.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:46.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:46.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:46.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:46.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:46.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:46.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:46.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:46.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:46.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:46.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:46.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:47.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:47.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:47.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:47.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 105397 105381 79.3356 52.8125 0.0179827 0.0125993 2023-10-09T06:24:47.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:47.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:47.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:47.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:47.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:47.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:47.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:47.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:47.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:47.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:47.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:47.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:47.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:47.935 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:47.936 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:47.936 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:48.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:48.049 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:48 smithi155 bash[59495]: cluster 2023-10-09T06:24:46.914128+0000 mgr.y (mgr.25723) 590 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:24:48.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:48.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:48 smithi125 bash[77299]: cluster 2023-10-09T06:24:46.914128+0000 mgr.y (mgr.25723) 590 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:24:48.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:48 smithi125 bash[80002]: cluster 2023-10-09T06:24:46.914128+0000 mgr.y (mgr.25723) 590 : cluster 0 pgmap v562: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:24:48.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:48.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 106213 106197 78.9983 51 0.0254747 0.0126527 2023-10-09T06:24:48.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:48.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:48.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:48.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:48.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:48.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:48.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:48.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:48.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:48.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:48.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:48.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:48.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:48.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:49.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:49.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:49.114 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:49 smithi155 bash[59495]: audit 2023-10-09T06:24:48.575844+0000 mon.a (mon.0) 792 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:49.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:49.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:49.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 107089 107073 78.713 54.75 0.0207972 0.0126988 2023-10-09T06:24:49.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:49.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:49.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:49.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:49 smithi125 bash[77299]: audit 2023-10-09T06:24:48.575844+0000 mon.a (mon.0) 792 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:49.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:49 smithi125 bash[80002]: audit 2023-10-09T06:24:48.575844+0000 mon.a (mon.0) 792 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:49.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:49.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:49.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:49.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:49.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:49.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:49.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:49.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:49.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:49.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:49.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:50.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:50.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:50.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:50 smithi155 bash[59495]: cluster 2023-10-09T06:24:48.914642+0000 mgr.y (mgr.25723) 591 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:24:50.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 107971 107955 78.4387 55.125 0.0237987 0.0127433 2023-10-09T06:24:50.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:50.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:50 smithi125 bash[77299]: cluster 2023-10-09T06:24:48.914642+0000 mgr.y (mgr.25723) 591 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:24:50.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:50 smithi125 bash[80002]: cluster 2023-10-09T06:24:48.914642+0000 mgr.y (mgr.25723) 591 : cluster 0 pgmap v563: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:24:50.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:50.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:50.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:50.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:50.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:50.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:50.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:50.848 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:50.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:50.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:50.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:51.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:51.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:51.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 108813 108797 78.142 52.625 0.0191033 0.0127911 2023-10-09T06:24:51.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:24:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:24:51] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:24:51.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:51.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:51.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:51.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:51.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:51.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:51.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:51.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:51.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:51.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:51.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:52.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:52.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:52 smithi155 bash[59495]: cluster 2023-10-09T06:24:50.915284+0000 mgr.y (mgr.25723) 592 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T06:24:52.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:52.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:52.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:52.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 109656 109640 77.8527 52.6875 0.00943454 0.0128399 2023-10-09T06:24:52.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:52.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:52.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:52.428 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:52 smithi125 bash[77299]: cluster 2023-10-09T06:24:50.915284+0000 mgr.y (mgr.25723) 592 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T06:24:52.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:52 smithi125 bash[80002]: cluster 2023-10-09T06:24:50.915284+0000 mgr.y (mgr.25723) 592 : cluster 0 pgmap v564: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s 2023-10-09T06:24:52.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:52.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:52.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:52.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:52.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:52.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:52.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:52.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:52.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:52.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:52.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:53.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:53.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:53.182 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 110461 110445 77.5432 50.3125 0.00488385 0.0128898 2023-10-09T06:24:53.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:53.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:53.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:53.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:53.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:53.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:53.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:53.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:53.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:53.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:53.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:53.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:53.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:54.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:54.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:54.068 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:54 smithi125 bash[77299]: cluster 2023-10-09T06:24:52.915944+0000 mgr.y (mgr.25723) 593 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T06:24:54.068 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:54 smithi125 bash[80002]: cluster 2023-10-09T06:24:52.915944+0000 mgr.y (mgr.25723) 593 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T06:24:54.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0099 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 111188 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 77.2054 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 30.2833 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 191 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 46 2023-10-09T06:24:54.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1235 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 484.532 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 3056 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 736 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0129479 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0108945 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 0.219491 2023-10-09T06:24:54.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00129902 2023-10-09T06:24:54.193 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:24:54.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:54.282 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:54 smithi155 bash[59495]: cluster 2023-10-09T06:24:52.915944+0000 mgr.y (mgr.25723) 593 : cluster 0 pgmap v565: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-10-09T06:24:54.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:54.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:54.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:54.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:54.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:54.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:54.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:54.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:54.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:54.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:54.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:54.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:54.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:55.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:55.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:55.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:55.254 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:55.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:55.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:55.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:55.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:55.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:55.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:55.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:55.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:55.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:55.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:55.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:55.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:55.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:56.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:56.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:56.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:56.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:56 smithi125 bash[77299]: cluster 2023-10-09T06:24:54.916469+0000 mgr.y (mgr.25723) 594 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 57 MiB/s wr, 925 op/s 2023-10-09T06:24:56.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:56 smithi125 bash[80002]: cluster 2023-10-09T06:24:54.916469+0000 mgr.y (mgr.25723) 594 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 57 MiB/s wr, 925 op/s 2023-10-09T06:24:56.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:56.406 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:56 smithi155 bash[59495]: cluster 2023-10-09T06:24:54.916469+0000 mgr.y (mgr.25723) 594 : cluster 0 pgmap v566: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 57 MiB/s wr, 925 op/s 2023-10-09T06:24:56.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:56.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:56.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:56.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:56.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:56.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:56.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:56.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:57.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:57.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:57.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:57.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:57.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:57.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:57.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:57.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:57.668 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:57.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:57.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:57.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:57.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:57.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:57.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:58.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:58 smithi125 bash[77299]: cluster 2023-10-09T06:24:56.917043+0000 mgr.y (mgr.25723) 595 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:24:58.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:58.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:58.221 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:58.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:58.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:58 smithi155 bash[59495]: cluster 2023-10-09T06:24:56.917043+0000 mgr.y (mgr.25723) 595 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:24:58.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:58.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:58 smithi125 bash[80002]: cluster 2023-10-09T06:24:56.917043+0000 mgr.y (mgr.25723) 595 : cluster 0 pgmap v567: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 47 MiB/s wr, 824 op/s 2023-10-09T06:24:58.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.481 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:58.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:58.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:58.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:58.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:58.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:58.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:58.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:58.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:59.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:59.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:59.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:59.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:59.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:59.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:59.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:59.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:59.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:59.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:59.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:59.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:59.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:59.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:59.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:59.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:24:59.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:00.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:00.062 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:24:59 smithi125 bash[77299]: cluster 2023-10-09T06:24:58.917555+0000 mgr.y (mgr.25723) 596 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 739 op/s 2023-10-09T06:25:00.063 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:24:59 smithi125 bash[80002]: cluster 2023-10-09T06:24:58.917555+0000 mgr.y (mgr.25723) 596 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 739 op/s 2023-10-09T06:25:00.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:24:59 smithi155 bash[59495]: cluster 2023-10-09T06:24:58.917555+0000 mgr.y (mgr.25723) 596 : cluster 0 pgmap v568: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 739 op/s 2023-10-09T06:25:00.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:00.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:00.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:00.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:00.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:00.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:00.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:00.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:00.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:00.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:00.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:00.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:00.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:00.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:01.015 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:01.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:01.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:01.252 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:01.252 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:01.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:01.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:01.340 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:25:01.383 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:25:01.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:01.422 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:25:01.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:01.456 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:25:01.471 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:01] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:25:01.503 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:25:01.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:01.554 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:25:01.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:01.605 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:25:01.651 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:25:01.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.655 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.656 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.656 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.656 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.656 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.656 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:25:01.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:01.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:01.773 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:25:01.773 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T06:25:01.773 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:25:01.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:01.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:01.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:01.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:01.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:01.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:02.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:02.117 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:01 smithi155 bash[59495]: cluster 2023-10-09T06:25:00.918152+0000 mgr.y (mgr.25723) 597 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 878 op/s 2023-10-09T06:25:02.117 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:01 smithi155 bash[59495]: audit 2023-10-09T06:25:01.271139+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.125:0/666300870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:02.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:02.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:02.245 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:01 smithi125 bash[77299]: cluster 2023-10-09T06:25:00.918152+0000 mgr.y (mgr.25723) 597 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 878 op/s 2023-10-09T06:25:02.245 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:01 smithi125 bash[77299]: audit 2023-10-09T06:25:01.271139+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.125:0/666300870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:02.246 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:01 smithi125 bash[80002]: cluster 2023-10-09T06:25:00.918152+0000 mgr.y (mgr.25723) 597 : cluster 0 pgmap v569: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 878 op/s 2023-10-09T06:25:02.246 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:01 smithi125 bash[80002]: audit 2023-10-09T06:25:01.271139+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.125:0/666300870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:02.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:02.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:02.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:02.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:02.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:02.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:02.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:02.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:02.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:02.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:02.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:02.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:02.998 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:25:03.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:03.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:02 smithi125 bash[77299]: audit 2023-10-09T06:25:02.122218+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.125:0/1523192942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:02 smithi125 bash[77299]: audit 2023-10-09T06:25:02.122929+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:02 smithi125 bash[80002]: audit 2023-10-09T06:25:02.122218+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.125:0/1523192942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:02 smithi125 bash[80002]: audit 2023-10-09T06:25:02.122929+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:03.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:03.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:02 smithi155 bash[59495]: audit 2023-10-09T06:25:02.122218+0000 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.125:0/1523192942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:02 smithi155 bash[59495]: audit 2023-10-09T06:25:02.122929+0000 mon.a (mon.0) 793 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:03.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:03.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:03.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:03.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:03.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:03.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:03.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:03.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:03.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:03.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:04.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:04.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:04.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:02.918981+0000 mgr.y (mgr.25723) 598 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 834 op/s 2023-10-09T06:25:04.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:02.972560+0000 mon.a (mon.0) 794 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:04.125 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:02.972675+0000 mon.a (mon.0) 795 : cluster 1 Cluster is now healthy 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: audit 2023-10-09T06:25:02.986207+0000 mon.a (mon.0) 796 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:02.986355+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:02.989998+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:03.001299+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: cluster 2023-10-09T06:25:03.003658+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: audit 2023-10-09T06:25:03.590674+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:04.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:03 smithi155 bash[59495]: audit 2023-10-09T06:25:03.593962+0000 mon.a (mon.0) 799 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:04.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:02.918981+0000 mgr.y (mgr.25723) 598 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 834 op/s 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:02.972560+0000 mon.a (mon.0) 794 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:02.972675+0000 mon.a (mon.0) 795 : cluster 1 Cluster is now healthy 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: audit 2023-10-09T06:25:02.986207+0000 mon.a (mon.0) 796 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:02.986355+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:02.989998+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:03.001299+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: cluster 2023-10-09T06:25:03.003658+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: audit 2023-10-09T06:25:03.590674+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:04.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[80002]: audit 2023-10-09T06:25:03.593962+0000 mon.a (mon.0) 799 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:02.918981+0000 mgr.y (mgr.25723) 598 : cluster 0 pgmap v570: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 834 op/s 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:02.972560+0000 mon.a (mon.0) 794 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:02.972675+0000 mon.a (mon.0) 795 : cluster 1 Cluster is now healthy 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: audit 2023-10-09T06:25:02.986207+0000 mon.a (mon.0) 796 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:02.986355+0000 mon.a (mon.0) 797 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:02.989998+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:03.001299+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: cluster 2023-10-09T06:25:03.003658+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e150 with expected crc 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: audit 2023-10-09T06:25:03.590674+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:04.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:03 smithi125 bash[77299]: audit 2023-10-09T06:25:03.593962+0000 mon.a (mon.0) 799 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:04.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:04.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:04.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:04.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:04.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:04.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:04.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:04.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:04.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:04.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:04.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:04.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:04.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:04.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:04.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:04.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:05.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:05.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:05.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:05.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:05.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:05.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:05.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:25:05.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:05.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:05.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:05.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:05.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:05.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:05.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:05.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:05.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:05.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:06.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:06.132 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:05 smithi155 bash[59495]: cluster 2023-10-09T06:25:04.919633+0000 mgr.y (mgr.25723) 599 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.7 MiB/s wr, 814 op/s 2023-10-09T06:25:06.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:06.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:06.276 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:05 smithi125 bash[80002]: cluster 2023-10-09T06:25:04.919633+0000 mgr.y (mgr.25723) 599 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.7 MiB/s wr, 814 op/s 2023-10-09T06:25:06.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:05 smithi125 bash[77299]: cluster 2023-10-09T06:25:04.919633+0000 mgr.y (mgr.25723) 599 : cluster 0 pgmap v572: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 5.7 MiB/s wr, 814 op/s 2023-10-09T06:25:06.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:06.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:06.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:06.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:06.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:06.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:06.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:06.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:06.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:06.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:07.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:07.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:07.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:07.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:07.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:07.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:07.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:07.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:07.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:07.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:07.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:07.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:07.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:07.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:07.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:07.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:08.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:08.078 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:25:08.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:25:08.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:07 smithi155 bash[59495]: cluster 2023-10-09T06:25:06.920085+0000 mgr.y (mgr.25723) 600 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 621 KiB/s wr, 812 op/s 2023-10-09T06:25:08.171 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:07 smithi125 bash[77299]: cluster 2023-10-09T06:25:06.920085+0000 mgr.y (mgr.25723) 600 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 621 KiB/s wr, 812 op/s 2023-10-09T06:25:08.172 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:07 smithi125 bash[80002]: cluster 2023-10-09T06:25:06.920085+0000 mgr.y (mgr.25723) 600 : cluster 0 pgmap v573: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 621 KiB/s wr, 812 op/s 2023-10-09T06:25:08.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:08.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:08.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:08.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:08.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:08.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:08.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:08.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:08.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:08.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:08.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:08.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:08.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:09.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.057 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:09.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:09.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:09.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:09.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:09.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:09.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:09.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:09.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:09.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:09.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.861 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:09 smithi125 bash[77299]: cluster 2023-10-09T06:25:08.920884+0000 mgr.y (mgr.25723) 601 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:25:09.861 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:09 smithi125 bash[80002]: cluster 2023-10-09T06:25:08.920884+0000 mgr.y (mgr.25723) 601 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:25:09.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:09.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:09.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:10.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:09 smithi155 bash[59495]: cluster 2023-10-09T06:25:08.920884+0000 mgr.y (mgr.25723) 601 : cluster 0 pgmap v574: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-09T06:25:10.159 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:10.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:10.267 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:10.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:10.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:10.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:10.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:10.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:10.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:10.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:10.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:10.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:10.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:10.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:10.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:11.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:11.107 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:25:11.107 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:25:11.107 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:25:11.108 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.377910 2023-10-09T06:25:11.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.3779104624652876 2023-10-09T06:25:11.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:11.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:11.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:11.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:11.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:11.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:11.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:11.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:11] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:25:11.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:11.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:11.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:11.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:12.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:12.021 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:11 smithi125 bash[77299]: cluster 2023-10-09T06:25:10.921556+0000 mgr.y (mgr.25723) 602 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T06:25:12.022 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:11 smithi125 bash[80002]: cluster 2023-10-09T06:25:10.921556+0000 mgr.y (mgr.25723) 602 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T06:25:12.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:12.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:12.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:12.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:12.305 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:12.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.313 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:12.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:12.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:11 smithi155 bash[59495]: cluster 2023-10-09T06:25:10.921556+0000 mgr.y (mgr.25723) 602 : cluster 0 pgmap v575: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-09T06:25:12.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:12.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:12.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:12.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:12.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:12.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:12.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:12.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:12.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:12.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:12.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:13.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:13.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:13.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:13.264 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:13.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:13.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:13.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:13.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:13.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:13.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:13.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:13.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:13.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:13.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:13.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:13.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:14.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:14.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:14.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:13 smithi155 bash[59495]: cluster 2023-10-09T06:25:12.922151+0000 mgr.y (mgr.25723) 603 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T06:25:14.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:13 smithi155 bash[59495]: audit 2023-10-09T06:25:13.878165+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:14.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:14.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:14.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:14.315 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:14 smithi125 bash[77299]: cluster 2023-10-09T06:25:12.922151+0000 mgr.y (mgr.25723) 603 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T06:25:14.316 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:14 smithi125 bash[77299]: audit 2023-10-09T06:25:13.878165+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:14.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:14 smithi125 bash[80002]: cluster 2023-10-09T06:25:12.922151+0000 mgr.y (mgr.25723) 603 : cluster 0 pgmap v576: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2023-10-09T06:25:14.316 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:14 smithi125 bash[80002]: audit 2023-10-09T06:25:13.878165+0000 mon.a (mon.0) 800 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:14.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:14.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:14.475 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:14.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:14.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:14.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:14.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:14.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:14.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:14.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:15.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:15.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:15.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:15.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:15.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:15.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:15.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:15.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:15.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:15.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:15.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:15.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:15.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:15.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:15.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:15.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:15.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:15.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:15.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:16.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:16.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:16.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:16.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:16 smithi125 bash[77299]: cluster 2023-10-09T06:25:14.922830+0000 mgr.y (mgr.25723) 604 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:16.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:16 smithi125 bash[80002]: cluster 2023-10-09T06:25:14.922830+0000 mgr.y (mgr.25723) 604 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:16.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:16.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:16.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:16.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:16.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:16 smithi155 bash[59495]: cluster 2023-10-09T06:25:14.922830+0000 mgr.y (mgr.25723) 604 : cluster 0 pgmap v577: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:16.417 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:16.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:16.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:16.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:16.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:16.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:16.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:16.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:16.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:16.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:17.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:17.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:17.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:17.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:17.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:17.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:17.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:17.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:17.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:17.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:17.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:17.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:17.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:17.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:17.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:17.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:18.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:18.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:18 smithi125 bash[77299]: cluster 2023-10-09T06:25:16.923309+0000 mgr.y (mgr.25723) 605 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:25:18.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:18 smithi125 bash[80002]: cluster 2023-10-09T06:25:16.923309+0000 mgr.y (mgr.25723) 605 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:25:18.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:18.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:18.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:18 smithi155 bash[59495]: cluster 2023-10-09T06:25:16.923309+0000 mgr.y (mgr.25723) 605 : cluster 0 pgmap v578: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:25:18.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:18.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:18.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:18.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:18.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:18.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:18.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:18.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:18.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:18.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:18.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:18.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:18.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:18.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:19.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:19.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:19.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:19.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:19.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:19.308 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:19 smithi125 bash[77299]: audit 2023-10-09T06:25:18.578774+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:19.308 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:19 smithi125 bash[80002]: audit 2023-10-09T06:25:18.578774+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:19.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:19.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:19 smithi155 bash[59495]: audit 2023-10-09T06:25:18.578774+0000 mon.a (mon.0) 801 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:19.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:19.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:19.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:19.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:19.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:19.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:19.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:19.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:19.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:20.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:20.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:20.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:20.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.301 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:20 smithi125 bash[80002]: cluster 2023-10-09T06:25:18.923713+0000 mgr.y (mgr.25723) 606 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T06:25:20.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:20 smithi125 bash[77299]: cluster 2023-10-09T06:25:18.923713+0000 mgr.y (mgr.25723) 606 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T06:25:20.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:20.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:20 smithi155 bash[59495]: cluster 2023-10-09T06:25:18.923713+0000 mgr.y (mgr.25723) 606 : cluster 0 pgmap v579: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-09T06:25:20.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:20.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:20.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:20.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:20.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:20.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:21.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:21.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:21.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:21] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-10-09T06:25:21.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:21.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:21.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[77299]: audit 2023-10-09T06:25:20.355456+0000 mon.a (mon.0) 802 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[77299]: audit 2023-10-09T06:25:20.369863+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[77299]: cluster 2023-10-09T06:25:20.924301+0000 mgr.y (mgr.25723) 607 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:25:21.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:21.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:21 smithi155 bash[59495]: audit 2023-10-09T06:25:20.355456+0000 mon.a (mon.0) 802 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:21 smithi155 bash[59495]: audit 2023-10-09T06:25:20.369863+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.407 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:21 smithi155 bash[59495]: cluster 2023-10-09T06:25:20.924301+0000 mgr.y (mgr.25723) 607 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:25:21.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:21.514 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[80002]: audit 2023-10-09T06:25:20.355456+0000 mon.a (mon.0) 802 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[80002]: audit 2023-10-09T06:25:20.369863+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:21.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:21 smithi125 bash[80002]: cluster 2023-10-09T06:25:20.924301+0000 mgr.y (mgr.25723) 607 : cluster 0 pgmap v580: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:25:21.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:21.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:21.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.820 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:25:21.820 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:25:21.820 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:25:21.820 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:25:21.820 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:25:21.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:21.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:21.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:21.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:22.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:22.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:22.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:22.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:22.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:22.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:22.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:22.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:22.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:22.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:22.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:22.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:22.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:22.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:22.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:22.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:23.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:23.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:23.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:23.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:23.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:23.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:23.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:23.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:23.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:23.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:23.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:23.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:23.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: audit 2023-10-09T06:25:22.697940+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: audit 2023-10-09T06:25:22.704745+0000 mon.a (mon.0) 805 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: cluster 2023-10-09T06:25:22.924863+0000 mgr.y (mgr.25723) 608 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:25:23.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: audit 2023-10-09T06:25:23.336050+0000 mon.a (mon.0) 806 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:23.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: audit 2023-10-09T06:25:23.336993+0000 mon.a (mon.0) 807 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[77299]: audit 2023-10-09T06:25:23.347354+0000 mon.a (mon.0) 808 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: audit 2023-10-09T06:25:22.697940+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: audit 2023-10-09T06:25:22.704745+0000 mon.a (mon.0) 805 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: cluster 2023-10-09T06:25:22.924863+0000 mgr.y (mgr.25723) 608 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: audit 2023-10-09T06:25:23.336050+0000 mon.a (mon.0) 806 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:23.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: audit 2023-10-09T06:25:23.336993+0000 mon.a (mon.0) 807 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:23.861 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:23 smithi125 bash[80002]: audit 2023-10-09T06:25:23.347354+0000 mon.a (mon.0) 808 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:23.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:23.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:23.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:24.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:24.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: audit 2023-10-09T06:25:22.697940+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:24.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: audit 2023-10-09T06:25:22.704745+0000 mon.a (mon.0) 805 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: cluster 2023-10-09T06:25:22.924863+0000 mgr.y (mgr.25723) 608 : cluster 0 pgmap v581: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:25:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: audit 2023-10-09T06:25:23.336050+0000 mon.a (mon.0) 806 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: audit 2023-10-09T06:25:23.336993+0000 mon.a (mon.0) 807 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:23 smithi155 bash[59495]: audit 2023-10-09T06:25:23.347354+0000 mon.a (mon.0) 808 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:24.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:24.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:24.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:24.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:24.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:24.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:24.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:24.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:24.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:24.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:24.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:24.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:24.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:24.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:24.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:25.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:25.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:25.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:25.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:25.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:25.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:25.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:25.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:25.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:25.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:25:25.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:25.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:25.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:25.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:25.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:25.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:25.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:25.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:26.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:26.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:26.052 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:25 smithi125 bash[77299]: cluster 2023-10-09T06:25:24.925534+0000 mgr.y (mgr.25723) 609 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2023-10-09T06:25:26.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:26.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:26.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:26.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:25 smithi155 bash[59495]: cluster 2023-10-09T06:25:24.925534+0000 mgr.y (mgr.25723) 609 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2023-10-09T06:25:26.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:26.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:26.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:26.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:26.441 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:26 smithi125 bash[80002]: cluster 2023-10-09T06:25:24.925534+0000 mgr.y (mgr.25723) 609 : cluster 0 pgmap v582: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2023-10-09T06:25:26.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:26.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:26.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:26.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:26.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:26.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:26.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:26.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:26.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:26.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:27.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:27.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:27.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:27.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:27.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:27.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:27.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:27.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:27.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:27.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:27.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:27.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:27.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:27.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:27.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:27.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:27.997 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:27 smithi125 bash[80002]: cluster 2023-10-09T06:25:26.926143+0000 mgr.y (mgr.25723) 610 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:25:27.997 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:27 smithi125 bash[80002]: audit 2023-10-09T06:25:27.943637+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.125:0/574066498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:27.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:27 smithi125 bash[77299]: cluster 2023-10-09T06:25:26.926143+0000 mgr.y (mgr.25723) 610 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:25:27.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:27 smithi125 bash[77299]: audit 2023-10-09T06:25:27.943637+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.125:0/574066498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:28.004 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:25:28.007 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:28.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:28.073 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:25:28.112 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:25:28.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:28.168 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:25:28.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:28.202 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:25:28.238 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:25:28.286 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:25:28.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:28.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.328 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:25:28.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:28.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:27 smithi155 bash[59495]: cluster 2023-10-09T06:25:26.926143+0000 mgr.y (mgr.25723) 610 : cluster 0 pgmap v583: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:25:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:27 smithi155 bash[59495]: audit 2023-10-09T06:25:27.943637+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.125:0/574066498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:28.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:28.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:28.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:28.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:28.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:28.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:28.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:28.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:28.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:28.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:28.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:28.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:29.007 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 0.37791 (8247662) 2023-10-09T06:25:29.010 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:25:29.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:29.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:29.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:28 smithi125 bash[77299]: audit 2023-10-09T06:25:28.092034+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.125:0/3769605020' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:28 smithi125 bash[77299]: audit 2023-10-09T06:25:28.093028+0000 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:28 smithi125 bash[77299]: audit 2023-10-09T06:25:28.682378+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.125:0/4111605651' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:28 smithi125 bash[77299]: audit 2023-10-09T06:25:28.683015+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:29 smithi125 bash[80002]: audit 2023-10-09T06:25:28.092034+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.125:0/3769605020' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:29 smithi125 bash[80002]: audit 2023-10-09T06:25:28.093028+0000 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:29 smithi125 bash[80002]: audit 2023-10-09T06:25:28.682378+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.125:0/4111605651' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:29 smithi125 bash[80002]: audit 2023-10-09T06:25:28.683015+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:29.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:29.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:29.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:29.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:29 smithi155 bash[59495]: audit 2023-10-09T06:25:28.092034+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.125:0/3769605020' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:29 smithi155 bash[59495]: audit 2023-10-09T06:25:28.093028+0000 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:29.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:29 smithi155 bash[59495]: audit 2023-10-09T06:25:28.682378+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.125:0/4111605651' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:29 smithi155 bash[59495]: audit 2023-10-09T06:25:28.683015+0000 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]: dispatch 2023-10-09T06:25:29.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:29.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:29.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:29.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:29.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:29.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:29.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:29.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:29.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:30.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:30.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:30.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:30.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: cluster 2023-10-09T06:25:28.926651+0000 mgr.y (mgr.25723) 611 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: cluster 2023-10-09T06:25:28.987544+0000 mon.a (mon.0) 812 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: audit 2023-10-09T06:25:28.994130+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: audit 2023-10-09T06:25:28.994563+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]': finished 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: cluster 2023-10-09T06:25:28.994839+0000 mon.a (mon.0) 815 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: cluster 2023-10-09T06:25:28.996058+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.161 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:30 smithi155 bash[59495]: cluster 2023-10-09T06:25:28.996773+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:30.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:30.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:30.290 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: cluster 2023-10-09T06:25:28.926651+0000 mgr.y (mgr.25723) 611 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:25:30.290 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: cluster 2023-10-09T06:25:28.987544+0000 mon.a (mon.0) 812 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:30.290 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: audit 2023-10-09T06:25:28.994130+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: audit 2023-10-09T06:25:28.994563+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]': finished 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: cluster 2023-10-09T06:25:28.994839+0000 mon.a (mon.0) 815 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: cluster 2023-10-09T06:25:28.996058+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[77299]: cluster 2023-10-09T06:25:28.996773+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: cluster 2023-10-09T06:25:28.926651+0000 mgr.y (mgr.25723) 611 : cluster 0 pgmap v584: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: cluster 2023-10-09T06:25:28.987544+0000 mon.a (mon.0) 812 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: audit 2023-10-09T06:25:28.994130+0000 mon.a (mon.0) 813 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: audit 2023-10-09T06:25:28.994563+0000 mon.a (mon.0) 814 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.3779104624652876}]': finished 2023-10-09T06:25:30.291 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: cluster 2023-10-09T06:25:28.994839+0000 mon.a (mon.0) 815 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:25:30.292 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: cluster 2023-10-09T06:25:28.996058+0000 osd.4 (osd.4) 77 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.292 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:30 smithi125 bash[80002]: cluster 2023-10-09T06:25:28.996773+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e151 with expected crc 2023-10-09T06:25:30.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:30.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:30.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:30.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:30.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:30.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:30.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:30.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:30.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:30.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:30.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:30.948 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:30.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:30.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:31.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:31.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:31.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:31.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:31 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.008889+0000 mon.a (mon.0) 816 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:25:31.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:31 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.015034+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:31 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.025499+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:31 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.025516+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.225 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:31 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.038933+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:31.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:31.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:31.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.008889+0000 mon.a (mon.0) 816 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:25:31.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.015034+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.025499+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.025516+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.038933+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:31] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.008889+0000 mon.a (mon.0) 816 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.015034+0000 osd.4 (osd.4) 78 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.025499+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.025516+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:31 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.038933+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e152 with expected crc 2023-10-09T06:25:31.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:31.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:31.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:31.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:31.643 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:31.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:31.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:31.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:31.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:31.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:31.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:25:32.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:32.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:32.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:32.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:32 smithi125 bash[80002]: cluster 2023-10-09T06:25:30.929141+0000 mgr.y (mgr.25723) 612 : cluster 0 pgmap v587: 89 pgs: 2 peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:25:32.156 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:32 smithi125 bash[80002]: cluster 2023-10-09T06:25:31.011014+0000 mon.a (mon.0) 817 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:25:32.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:32 smithi125 bash[77299]: cluster 2023-10-09T06:25:30.929141+0000 mgr.y (mgr.25723) 612 : cluster 0 pgmap v587: 89 pgs: 2 peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:25:32.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:32 smithi125 bash[77299]: cluster 2023-10-09T06:25:31.011014+0000 mon.a (mon.0) 817 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:25:32.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:32.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:32.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:32.309 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:32 smithi155 bash[59495]: cluster 2023-10-09T06:25:30.929141+0000 mgr.y (mgr.25723) 612 : cluster 0 pgmap v587: 89 pgs: 2 peering, 87 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:25:32.309 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:32 smithi155 bash[59495]: cluster 2023-10-09T06:25:31.011014+0000 mon.a (mon.0) 817 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:25:32.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:32.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:32.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:32.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:32.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:32.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:32.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:32.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:32.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:32.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:32.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:32.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:32.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:33.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:33.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:33.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:33.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:33.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:33.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:33.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:33.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:25:33.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:33.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:33.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:33.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:33.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:33.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:33.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:33.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:33.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:33.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:33.935 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:33.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:34.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:34.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:34.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[77299]: cluster 2023-10-09T06:25:32.933512+0000 mgr.y (mgr.25723) 613 : cluster 0 pgmap v588: 89 pgs: 2 peering, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:25:34.120 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[77299]: audit 2023-10-09T06:25:33.586820+0000 mon.a (mon.0) 818 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:34.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[77299]: audit 2023-10-09T06:25:33.588115+0000 mon.a (mon.0) 819 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:34.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[80002]: cluster 2023-10-09T06:25:32.933512+0000 mgr.y (mgr.25723) 613 : cluster 0 pgmap v588: 89 pgs: 2 peering, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:25:34.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[80002]: audit 2023-10-09T06:25:33.586820+0000 mon.a (mon.0) 818 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:34.121 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:34 smithi125 bash[80002]: audit 2023-10-09T06:25:33.588115+0000 mon.a (mon.0) 819 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:34.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:34.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:34.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:34.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:34.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:34.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:34 smithi155 bash[59495]: cluster 2023-10-09T06:25:32.933512+0000 mgr.y (mgr.25723) 613 : cluster 0 pgmap v588: 89 pgs: 2 peering, 87 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:25:34.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:34 smithi155 bash[59495]: audit 2023-10-09T06:25:33.586820+0000 mon.a (mon.0) 818 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:34.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:34 smithi155 bash[59495]: audit 2023-10-09T06:25:33.588115+0000 mon.a (mon.0) 819 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:34.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:34.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:34.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:34.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:34.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:34.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:34.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:34.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:34.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:34.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:34.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:35.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:35.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:35.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:35.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.358 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:35 smithi125 bash[77299]: cluster 2023-10-09T06:25:35.024107+0000 mon.a (mon.0) 820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:25:35.358 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:35 smithi125 bash[80002]: cluster 2023-10-09T06:25:35.024107+0000 mon.a (mon.0) 820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:25:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:35.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:35 smithi155 bash[59495]: cluster 2023-10-09T06:25:35.024107+0000 mon.a (mon.0) 820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:25:35.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:35.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:35.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:35.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:35.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:35.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:35.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:35.970 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:36.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:36.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:36.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:36.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:36.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:36.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:36.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:36.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:36 smithi125 bash[77299]: cluster 2023-10-09T06:25:34.934066+0000 mgr.y (mgr.25723) 614 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T06:25:36.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:36 smithi125 bash[80002]: cluster 2023-10-09T06:25:34.934066+0000 mgr.y (mgr.25723) 614 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T06:25:36.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:36 smithi155 bash[59495]: cluster 2023-10-09T06:25:34.934066+0000 mgr.y (mgr.25723) 614 : cluster 0 pgmap v589: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2023-10-09T06:25:36.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:36.430 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:25:36.430 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:25:36.430 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:25:36.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T06:25:36.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:36.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:36.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:36.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:36.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:36.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:36.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:36.882 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:36.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:36.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:36.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:37.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:37.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:37.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:37.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:37.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:37.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:37.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:37.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:37.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:37.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:37.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:37.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:37.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:38.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:38.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:38 smithi155 bash[59495]: cluster 2023-10-09T06:25:36.934716+0000 mgr.y (mgr.25723) 615 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:38.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:38.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:38.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:38.325 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:38 smithi125 bash[77299]: cluster 2023-10-09T06:25:36.934716+0000 mgr.y (mgr.25723) 615 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:38.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:38 smithi125 bash[80002]: cluster 2023-10-09T06:25:36.934716+0000 mgr.y (mgr.25723) 615 : cluster 0 pgmap v590: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:25:38.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:38.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:38.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:38.521 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:38.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:38.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:38.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:38.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:38.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:38.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:39.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:39.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.156 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:39.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:39.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:39.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:39.421 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:39.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:39.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:39.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:39.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:39.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:39 smithi155 bash[59495]: cluster 2023-10-09T06:25:38.935184+0000 mgr.y (mgr.25723) 616 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-09T06:25:39.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.946 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:39 smithi125 bash[77299]: cluster 2023-10-09T06:25:38.935184+0000 mgr.y (mgr.25723) 616 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-09T06:25:39.946 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:39 smithi125 bash[80002]: cluster 2023-10-09T06:25:38.935184+0000 mgr.y (mgr.25723) 616 : cluster 0 pgmap v591: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-09T06:25:39.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:39.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:40.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:40.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:40.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:40.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:40.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:40.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:40.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:40.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:40.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:40.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:40.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:40.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:40.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:41.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:41.041 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:41.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:41.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:41.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:41.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:41.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:41] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-10-09T06:25:41.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:41.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:41.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:41.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:41.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:41.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:41.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:41.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:41.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:41.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:42.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:42.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:42.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:42 smithi125 bash[77299]: cluster 2023-10-09T06:25:40.935643+0000 mgr.y (mgr.25723) 617 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:25:42.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:42 smithi155 bash[59495]: cluster 2023-10-09T06:25:40.935643+0000 mgr.y (mgr.25723) 617 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:25:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:42.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:42.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:42.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:42 smithi125 bash[80002]: cluster 2023-10-09T06:25:40.935643+0000 mgr.y (mgr.25723) 617 : cluster 0 pgmap v592: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:25:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:42.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:42.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:42.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:42.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:42.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:42.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:42.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:42.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:42.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:42.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:43.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:43.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:43.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:43.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:43.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:43.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:43.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:43.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:43.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:43.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:43.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:43.997 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:43.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:44.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:44.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:44.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:44.245 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:44 smithi125 bash[77299]: cluster 2023-10-09T06:25:42.936062+0000 mgr.y (mgr.25723) 618 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:25:44.245 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:44 smithi125 bash[77299]: audit 2023-10-09T06:25:43.912712+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:44.245 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:44 smithi125 bash[80002]: cluster 2023-10-09T06:25:42.936062+0000 mgr.y (mgr.25723) 618 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:25:44.245 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:44 smithi125 bash[80002]: audit 2023-10-09T06:25:43.912712+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:44.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:44.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:44.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:44 smithi155 bash[59495]: cluster 2023-10-09T06:25:42.936062+0000 mgr.y (mgr.25723) 618 : cluster 0 pgmap v593: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:25:44.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:44 smithi155 bash[59495]: audit 2023-10-09T06:25:43.912712+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:44.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:44.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:44.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:44.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:44.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:44.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:44.787 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:44.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:44.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:44.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:44.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:45.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:45.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:45.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:45.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:45.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:45.190 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:45.190 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:45.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:45.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:45.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:45.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:45.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:45.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:45.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:45.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:45.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:45.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:45.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:45.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:45.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:45.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:45.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:46.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:46.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:46.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:46.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:46 smithi125 bash[80002]: cluster 2023-10-09T06:25:44.936677+0000 mgr.y (mgr.25723) 619 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:25:46.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:46 smithi125 bash[77299]: cluster 2023-10-09T06:25:44.936677+0000 mgr.y (mgr.25723) 619 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:25:46.249 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:46.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:46.365 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:46 smithi155 bash[59495]: cluster 2023-10-09T06:25:44.936677+0000 mgr.y (mgr.25723) 619 : cluster 0 pgmap v594: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:25:46.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:46.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:46.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:46.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:46.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:46.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:46.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:46.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:46.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:46.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:46.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:47.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:47.037 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:47.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:47.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:47.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:47.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:47.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:47.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:47.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:47.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:47.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:47.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:47.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:47.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:47.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:47.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:47.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:47.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:47.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:47.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:48.017 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:48 smithi155 bash[59495]: cluster 2023-10-09T06:25:46.937242+0000 mgr.y (mgr.25723) 620 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:25:48.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:48.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:48 smithi125 bash[77299]: cluster 2023-10-09T06:25:46.937242+0000 mgr.y (mgr.25723) 620 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:25:48.114 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:48 smithi125 bash[80002]: cluster 2023-10-09T06:25:46.937242+0000 mgr.y (mgr.25723) 620 : cluster 0 pgmap v595: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:25:48.114 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:48.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:48.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:48.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:48.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:48.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:48.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:48.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:48.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:48.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:48.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:49.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:49.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:49.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:25:49.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:49.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:49.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:49.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:49.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:49.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:49.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:49.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:49.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:49.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:49.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.830 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:49 smithi155 bash[59495]: audit 2023-10-09T06:25:48.584218+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:49.830 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:49 smithi155 bash[59495]: audit 2023-10-09T06:25:48.586332+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:49.830 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:49 smithi155 bash[59495]: cluster 2023-10-09T06:25:48.937724+0000 mgr.y (mgr.25723) 621 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-09T06:25:49.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:49.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[77299]: audit 2023-10-09T06:25:48.584218+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:49.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[77299]: audit 2023-10-09T06:25:48.586332+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:49.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[77299]: cluster 2023-10-09T06:25:48.937724+0000 mgr.y (mgr.25723) 621 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-09T06:25:49.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[80002]: audit 2023-10-09T06:25:48.584218+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:25:49.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[80002]: audit 2023-10-09T06:25:48.586332+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:49.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:49 smithi125 bash[80002]: cluster 2023-10-09T06:25:48.937724+0000 mgr.y (mgr.25723) 621 : cluster 0 pgmap v596: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-09T06:25:49.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:49.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:49.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:50.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:50.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:50.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:50.241 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:50.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:50.371 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:50.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:50.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:50.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:50.587 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:25:50.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:50.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:25:50.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:50.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:50.663 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:25:50.712 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:25:50.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:50.757 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:25:50.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:50.799 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:25:50.844 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:25:50.883 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:25:50.885 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.886 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:25:50.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:50.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:50.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:51.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:51.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:50 smithi125 bash[77299]: audit 2023-10-09T06:25:50.539082+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.125:0/3675149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:51.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:50 smithi125 bash[80002]: audit 2023-10-09T06:25:50.539082+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.125:0/3675149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:51.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:50 smithi155 bash[59495]: audit 2023-10-09T06:25:50.539082+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.125:0/3675149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:51.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:51.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:51.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:51.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:25:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:25:51] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:25:51.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:51.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:51.615 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:51.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:51.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:51.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.978 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:51 smithi125 bash[80002]: cluster 2023-10-09T06:25:50.938310+0000 mgr.y (mgr.25723) 622 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:25:51.978 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:51 smithi125 bash[77299]: cluster 2023-10-09T06:25:50.938310+0000 mgr.y (mgr.25723) 622 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:25:51.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:51.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:52.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.107 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:51 smithi155 bash[59495]: cluster 2023-10-09T06:25:50.938310+0000 mgr.y (mgr.25723) 622 : cluster 0 pgmap v597: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:25:52.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:52.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:52.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:52.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:52.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:52.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:52.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:52.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:52.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:52.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:52.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:52.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:52.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:52.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:53.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:53.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:53.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:53.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:53.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:53.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:53.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:53.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:53.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:53.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:53.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:53.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:53.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:53.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:53.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:53.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:53.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:54.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:54.069 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:53 smithi125 bash[77299]: cluster 2023-10-09T06:25:52.938879+0000 mgr.y (mgr.25723) 623 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:25:54.069 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:53 smithi125 bash[80002]: cluster 2023-10-09T06:25:52.938879+0000 mgr.y (mgr.25723) 623 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:25:54.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:54.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.219 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:53 smithi155 bash[59495]: cluster 2023-10-09T06:25:52.938879+0000 mgr.y (mgr.25723) 623 : cluster 0 pgmap v598: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:25:54.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:54.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:54.480 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:54.561 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:25:54.561 INFO:teuthology.orchestra.run.smithi125.stdout:avg 33.375 2023-10-09T06:25:54.561 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 5.09749 -> 5.09749 (expected baseline 5.40399) 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.5 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 0.05 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0208 2023-10-09T06:25:54.562 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.2077 2023-10-09T06:25:54.566 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:25:54.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:54.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:54.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:54.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:54.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:54.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:54.929 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:54.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:55.018 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:25:55.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:55.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:55 smithi125 bash[77299]: audit 2023-10-09T06:25:54.398276+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.125:0/3074798962' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.057 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:55 smithi125 bash[77299]: audit 2023-10-09T06:25:54.399394+0000 mon.a (mon.0) 824 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:55 smithi125 bash[80002]: audit 2023-10-09T06:25:54.398276+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.125:0/3074798962' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.058 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:55 smithi125 bash[80002]: audit 2023-10-09T06:25:54.399394+0000 mon.a (mon.0) 824 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:55.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:55.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:55.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:55.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:55.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:55 smithi155 bash[59495]: audit 2023-10-09T06:25:54.398276+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.125:0/3074798962' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:55 smithi155 bash[59495]: audit 2023-10-09T06:25:54.399394+0000 mon.a (mon.0) 824 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:55.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:55.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:55.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:55.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:25:55.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:55.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:55.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:55.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:55.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:55.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:55.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:55.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:55.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:56.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:56.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:56.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:56.232 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:56.242 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: audit 2023-10-09T06:25:54.553838+0000 mgr.y (mgr.25723) 624 : audit 0 from='client.45020 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:54.939755+0000 mgr.y (mgr.25723) 625 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:54.994396+0000 mon.a (mon.0) 825 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: audit 2023-10-09T06:25:55.006293+0000 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.006472+0000 mon.a (mon.0) 827 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.007463+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.010781+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.014167+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.015675+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.244 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:56 smithi155 bash[59495]: cluster 2023-10-09T06:25:55.021662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:56.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:56.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: audit 2023-10-09T06:25:54.553838+0000 mgr.y (mgr.25723) 624 : audit 0 from='client.45020 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:25:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:54.939755+0000 mgr.y (mgr.25723) 625 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T06:25:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:54.994396+0000 mon.a (mon.0) 825 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: audit 2023-10-09T06:25:55.006293+0000 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.006472+0000 mon.a (mon.0) 827 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.007463+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.010781+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.014167+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.015675+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[77299]: cluster 2023-10-09T06:25:55.021662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: audit 2023-10-09T06:25:54.553838+0000 mgr.y (mgr.25723) 624 : audit 0 from='client.45020 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:54.939755+0000 mgr.y (mgr.25723) 625 : cluster 0 pgmap v599: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:54.994396+0000 mon.a (mon.0) 825 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: audit 2023-10-09T06:25:55.006293+0000 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.006472+0000 mon.a (mon.0) 827 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.007463+0000 osd.4 (osd.4) 79 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.010781+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.014167+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.015675+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:56 smithi125 bash[80002]: cluster 2023-10-09T06:25:55.021662+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e153 with expected crc 2023-10-09T06:25:56.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:56.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:56.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:56.666 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:25:56.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T06:25:56.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:56.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:56.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:56.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:56.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:56.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:56.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:57.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:57.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:57.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:57.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:57.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:57.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:57.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:57.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:57.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:57.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:57.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:57.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:58.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:58.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:58.163 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:58.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:58.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:58.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:58.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:58 smithi125 bash[77299]: cluster 2023-10-09T06:25:56.940282+0000 mgr.y (mgr.25723) 626 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:25:58.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:58 smithi125 bash[80002]: cluster 2023-10-09T06:25:56.940282+0000 mgr.y (mgr.25723) 626 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:25:58.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:58.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:58 smithi155 bash[59495]: cluster 2023-10-09T06:25:56.940282+0000 mgr.y (mgr.25723) 626 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:25:58.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:58.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:58.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:58.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:58.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:58.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:58.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:58.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:58.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:59.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:59.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:59.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:25:59.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:59.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:59.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:59.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:59.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:59.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:59.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:59.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:59.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:59.838 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:25:59 smithi155 bash[59495]: cluster 2023-10-09T06:25:58.940873+0000 mgr.y (mgr.25723) 627 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:25:59.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:59.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:59.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:59.982 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:25:59 smithi125 bash[77299]: cluster 2023-10-09T06:25:58.940873+0000 mgr.y (mgr.25723) 627 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:25:59.982 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:25:59 smithi125 bash[80002]: cluster 2023-10-09T06:25:58.940873+0000 mgr.y (mgr.25723) 627 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:25:59.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:25:59.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:00.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:00.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:00.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:00.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:00.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:00.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:00.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:00.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:00.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:00.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:00.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:00.787 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:00.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:00.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:00.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:01.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:01.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:01.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:01.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:01.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:01] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:26:01.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:01.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:01.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:01.472 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:01.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:01.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:01.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:01.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:01.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:01.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:01.808 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:01.813 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:01.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:01.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:01.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:01.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:02.027 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:02.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:01 smithi125 bash[77299]: cluster 2023-10-09T06:26:00.941521+0000 mgr.y (mgr.25723) 628 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:02.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:01 smithi125 bash[80002]: cluster 2023-10-09T06:26:00.941521+0000 mgr.y (mgr.25723) 628 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:02.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:02.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:02.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:02 smithi155 bash[59495]: cluster 2023-10-09T06:26:00.941521+0000 mgr.y (mgr.25723) 628 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:02.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:02.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:02.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:02.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:02.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:02.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:02.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:02.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:02.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:02.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:02.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:02.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:02.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:02.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:03.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:03.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:03.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:03.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:03.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:03.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:03.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:03.485 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:03.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:03.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:03.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:03.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:03.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:03.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:03.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:03.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:04.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:04.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:04.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:04.183 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:04 smithi155 bash[59495]: cluster 2023-10-09T06:26:02.942166+0000 mgr.y (mgr.25723) 629 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:04.183 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:04 smithi155 bash[59495]: audit 2023-10-09T06:26:03.577163+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:04.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:04.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:04.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:04.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:04.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:03 smithi125 bash[77299]: cluster 2023-10-09T06:26:02.942166+0000 mgr.y (mgr.25723) 629 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:04.373 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:03 smithi125 bash[77299]: audit 2023-10-09T06:26:03.577163+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:04.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:03 smithi125 bash[80002]: cluster 2023-10-09T06:26:02.942166+0000 mgr.y (mgr.25723) 629 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:04.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:03 smithi125 bash[80002]: audit 2023-10-09T06:26:03.577163+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:04.401 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:04.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:04.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:04.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162855 6.8 e153/142 6:150f72d0:::benchmark_data_smithi125_130224_object51664:head v 153'10115, mlcod=153'10113)", 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.564562+0000", 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.043697547000000003, 2023-10-09T06:26:04.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.043722221999999998, 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564562+0000", 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564562+0000", 2023-10-09T06:26:04.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564561+0000", 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564574+0000", 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2883e-05 2023-10-09T06:26:04.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564587+0000", 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2508e-05 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564595+0000", 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5339999999999999e-06 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564671+0000", 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6049000000000003e-05 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.564692+0000", 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0386000000000001e-05 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.579877+0000", 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015185086 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162853 6.1 e153/142 6:83fcee42:::benchmark_data_smithi125_130224_object51662:head v 153'10133, mlcod=153'10129)", 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.569100+0000", 2023-10-09T06:26:04.619 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.039159721000000002, 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.039232613, 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569100+0000", 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569100+0000", 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569099+0000", 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569107+0000", 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1089999999999998e-06 2023-10-09T06:26:04.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569119+0000", 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1494999999999999e-05 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569127+0000", 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.678e-06 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569190+0000", 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3557000000000002e-05 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.569210+0000", 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9409999999999999e-05 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162857 6.8 e153/142 6:11db3407:::benchmark_data_smithi125_130224_object51666:head v 153'10116, mlcod=153'10113)", 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.571225+0000", 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.037035150000000003, 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.037145804999999997, 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571225+0000", 2023-10-09T06:26:04.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571225+0000", 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571224+0000", 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.627 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571230+0000", 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1299999999999998e-06 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571240+0000", 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8980000000000008e-06 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571247+0000", 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4010000000000002e-06 2023-10-09T06:26:04.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571337+0000", 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0735000000000005e-05 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.571356+0000", 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8763999999999999e-05 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.629 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162864 6.0 e153/142 6:023a7a03:::benchmark_data_smithi125_130224_object51673:head v 153'10221, mlcod=153'10219)", 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.574995+0000", 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.033265322999999999, 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.033413186999999997, 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.574995+0000", 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.574995+0000", 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.574994+0000", 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.575002+0000", 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4689999999999997e-06 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.575015+0000", 2023-10-09T06:26:04.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2516e-05 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.575024+0000", 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9309999999999997e-06 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576859+0000", 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018358179999999999 2023-10-09T06:26:04.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576881+0000", 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1189999999999999e-05 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162867 6.e e153/142 6:742b8232:::benchmark_data_smithi125_130224_object51676:head v 153'10145, mlcod=153'10143)", 2023-10-09T06:26:04.634 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.576927+0000", 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.031332895999999999, 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.031518417999999999, 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576927+0000", 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576927+0000", 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576926+0000", 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576933+0000", 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.34e-06 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576945+0000", 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1790000000000001e-05 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.576952+0000", 2023-10-09T06:26:04.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4789999999999999e-06 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577073+0000", 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120357 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577229+0000", 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015621600000000001 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.638 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:162868 6.2 e153/142)", 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:26:04.577141+0000", 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.031118907000000001, 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.031338236999999998, 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:04.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577141+0000", 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577141+0000", 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:04.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577140+0000", 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577146+0000", 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577154+0000", 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0579999999999998e-06 2023-10-09T06:26:04.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:26:04.577160+0000", 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7479999999999999e-06 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T06:26:04.642 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 6 2023-10-09T06:26:04.643 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:26:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:04.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:04.744 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:04.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:04.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:04.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:04.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:05.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:05.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:05.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:05.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:05.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:05.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:05.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:05.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:05.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:05.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:05.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:05.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:05.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:05.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:05.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:06.007 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:06 smithi155 bash[59495]: cluster 2023-10-09T06:26:04.942815+0000 mgr.y (mgr.25723) 630 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:26:06.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:06.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:06 smithi125 bash[77299]: cluster 2023-10-09T06:26:04.942815+0000 mgr.y (mgr.25723) 630 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:26:06.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:06 smithi125 bash[80002]: cluster 2023-10-09T06:26:04.942815+0000 mgr.y (mgr.25723) 630 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:26:06.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:06.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:06.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:06.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:06.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:06.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:06.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:06.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:06.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:06.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:06.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:06.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:06.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:07.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:07.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:07.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:07.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:07.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:07.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:07.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:07.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:07.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:07.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:07.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:26:07.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:07.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:07.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:07.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:07.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:07.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:08.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:08.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.120 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:08 smithi155 bash[59495]: cluster 2023-10-09T06:26:06.943347+0000 mgr.y (mgr.25723) 631 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:26:08.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:08.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:08.284 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:08.289 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:08 smithi125 bash[77299]: cluster 2023-10-09T06:26:06.943347+0000 mgr.y (mgr.25723) 631 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:26:08.289 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:08 smithi125 bash[80002]: cluster 2023-10-09T06:26:06.943347+0000 mgr.y (mgr.25723) 631 : cluster 0 pgmap v606: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:26:08.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:08.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:08.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:08.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:08.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:08.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:08.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:08.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:08.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:08.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:08.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:09.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:09.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:09.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:09.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:09.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:09.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:09.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:09.488 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:09.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:09.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:09.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:09.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:09.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:09.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:09.892 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:09 smithi155 bash[59495]: cluster 2023-10-09T06:26:08.943774+0000 mgr.y (mgr.25723) 632 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:09.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:09.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:10.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:09 smithi125 bash[77299]: cluster 2023-10-09T06:26:08.943774+0000 mgr.y (mgr.25723) 632 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:09 smithi125 bash[80002]: cluster 2023-10-09T06:26:08.943774+0000 mgr.y (mgr.25723) 632 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:10.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:10.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:10.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:10.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.444 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:10.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:10.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:10.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:10.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:10.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:10.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:10.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:10.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:10.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:11.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:11.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:11.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:11.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:11.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:11.394 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:11.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:11.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:11.540 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:11] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.33.4" 2023-10-09T06:26:11.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:11.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:11.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:11.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:11.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:11.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:11.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:11.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:12.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:12.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:12.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:12 smithi125 bash[77299]: cluster 2023-10-09T06:26:10.944373+0000 mgr.y (mgr.25723) 633 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:26:12.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:12 smithi125 bash[80002]: cluster 2023-10-09T06:26:10.944373+0000 mgr.y (mgr.25723) 633 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:26:12.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:12.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:12.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:12.330 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:12 smithi155 bash[59495]: cluster 2023-10-09T06:26:10.944373+0000 mgr.y (mgr.25723) 633 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:26:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:12.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:12.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:12.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:12.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:12.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:12.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:12.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:12.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:12.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:12.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:13.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:13.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:13.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:13.137 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:13.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:13.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:13.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:13.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:13.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:13.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:13.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:13.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:13.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:13.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:13.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:13.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:13.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:13.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:13.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:14.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:14.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:14.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:14.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:14.229 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:14 smithi155 bash[59495]: cluster 2023-10-09T06:26:12.944861+0000 mgr.y (mgr.25723) 634 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T06:26:14.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:14.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:14 smithi125 bash[77299]: cluster 2023-10-09T06:26:12.944861+0000 mgr.y (mgr.25723) 634 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T06:26:14.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:14 smithi125 bash[80002]: cluster 2023-10-09T06:26:12.944861+0000 mgr.y (mgr.25723) 634 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-09T06:26:14.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:14.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:14.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:14.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:14.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:14.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:14.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:14.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:14.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:avg 33.375 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 5.09749 -> 5.09749 (expected baseline 5.40399) 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.5 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:26:14.832 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 3 2023-10-09T06:26:14.833 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:26:14.833 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0172 2023-10-09T06:26:14.833 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.1717 2023-10-09T06:26:14.838 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:26:14.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:14.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:14.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:15.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:15.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.221 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:15.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:15.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:15.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:15.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:15.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:15.742 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:26:15.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:15.781 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:26:15.850 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:26:15.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:15.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:15.947 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:26:15.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:15.985 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:26:16.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:16.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:16.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:26:16.059 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:26:16.093 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.100 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:26:16.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:16.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:16.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:16.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:16.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[77299]: audit 2023-10-09T06:26:14.826141+0000 mgr.y (mgr.25723) 635 : audit 0 from='client.45023 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:16.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[77299]: cluster 2023-10-09T06:26:14.945475+0000 mgr.y (mgr.25723) 636 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:26:16.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[77299]: audit 2023-10-09T06:26:15.686641+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.125:0/3388057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:16.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[80002]: audit 2023-10-09T06:26:14.826141+0000 mgr.y (mgr.25723) 635 : audit 0 from='client.45023 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:16.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[80002]: cluster 2023-10-09T06:26:14.945475+0000 mgr.y (mgr.25723) 636 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:26:16.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:16 smithi125 bash[80002]: audit 2023-10-09T06:26:15.686641+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.125:0/3388057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:16.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:16.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:16 smithi155 bash[59495]: audit 2023-10-09T06:26:14.826141+0000 mgr.y (mgr.25723) 635 : audit 0 from='client.45023 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:16 smithi155 bash[59495]: cluster 2023-10-09T06:26:14.945475+0000 mgr.y (mgr.25723) 636 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:26:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:16 smithi155 bash[59495]: audit 2023-10-09T06:26:15.686641+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.125:0/3388057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:16.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:16.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:16.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:16.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:16.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:16.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:16.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:16.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:16.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:16.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:16.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:17.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:17.100 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:17.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:17.206 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:17.238 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:17.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:17.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:17.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:17.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:17.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:17.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:17.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:17.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:17.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:17.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:17.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:17.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:17.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:17.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:17.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:18.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:18.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:18.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:18.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:18.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:18.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:18.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:18 smithi155 bash[59495]: cluster 2023-10-09T06:26:16.945950+0000 mgr.y (mgr.25723) 637 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:18.295 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:26:18.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T06:26:18.299 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:18 smithi125 bash[80002]: cluster 2023-10-09T06:26:16.945950+0000 mgr.y (mgr.25723) 637 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:18.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:18 smithi125 bash[77299]: cluster 2023-10-09T06:26:16.945950+0000 mgr.y (mgr.25723) 637 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:18.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:18.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:18.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:18.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:18.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:18.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:18.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:18.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:18.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:18.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:18.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:18.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:19.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:19.107 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:19 smithi155 bash[59495]: audit 2023-10-09T06:26:18.577610+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:19.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:19.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:19.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:19.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:19 smithi125 bash[77299]: audit 2023-10-09T06:26:18.577610+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:19.286 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:19 smithi125 bash[80002]: audit 2023-10-09T06:26:18.577610+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:19.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:19.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:19.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:19.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:19.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:19.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:19.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:19.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:19.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:19.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:20.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:20.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:20.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:20.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[77299]: cluster 2023-10-09T06:26:18.952639+0000 mgr.y (mgr.25723) 638 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:20.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[77299]: audit 2023-10-09T06:26:19.802181+0000 mon.a (mon.0) 830 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.203 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[77299]: audit 2023-10-09T06:26:19.806431+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.125:0/2977347372' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[80002]: cluster 2023-10-09T06:26:18.952639+0000 mgr.y (mgr.25723) 638 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:20.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[80002]: audit 2023-10-09T06:26:19.802181+0000 mon.a (mon.0) 830 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:20 smithi125 bash[80002]: audit 2023-10-09T06:26:19.806431+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.125:0/2977347372' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:20.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:20.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:20.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:20 smithi155 bash[59495]: cluster 2023-10-09T06:26:18.952639+0000 mgr.y (mgr.25723) 638 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:26:20.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:20 smithi155 bash[59495]: audit 2023-10-09T06:26:19.802181+0000 mon.a (mon.0) 830 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:20 smithi155 bash[59495]: audit 2023-10-09T06:26:19.806431+0000 mon.b (mon.1) 12 : audit 1 from='client.? 172.21.15.125:0/2977347372' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:20.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:20.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:20.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:20.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:20.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:20.772 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:26:20.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:20.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:20.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:20.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:20.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:20.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:21.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.032 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:21 smithi155 bash[59495]: cluster 2023-10-09T06:26:20.742925+0000 mon.a (mon.0) 831 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:21.032 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:21 smithi155 bash[59495]: audit 2023-10-09T06:26:20.751818+0000 mon.a (mon.0) 832 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:21.032 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:21 smithi155 bash[59495]: cluster 2023-10-09T06:26:20.751891+0000 mon.a (mon.0) 833 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:26:21.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:21.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:26:21.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[77299]: cluster 2023-10-09T06:26:20.742925+0000 mon.a (mon.0) 831 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:21.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[77299]: audit 2023-10-09T06:26:20.751818+0000 mon.a (mon.0) 832 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:21.170 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[77299]: cluster 2023-10-09T06:26:20.751891+0000 mon.a (mon.0) 833 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:26:21.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[80002]: cluster 2023-10-09T06:26:20.742925+0000 mon.a (mon.0) 831 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:21.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[80002]: audit 2023-10-09T06:26:20.751818+0000 mon.a (mon.0) 832 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:21.170 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[80002]: cluster 2023-10-09T06:26:20.751891+0000 mon.a (mon.0) 833 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:26:21.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:21.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:21.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:21.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:21.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:21.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:21.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:21.639 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:21] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:26:21.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:21.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:21.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:21.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:21.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:21.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:22.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:22.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[77299]: cluster 2023-10-09T06:26:20.755110+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[77299]: cluster 2023-10-09T06:26:20.759195+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[77299]: cluster 2023-10-09T06:26:20.953013+0000 mgr.y (mgr.25723) 639 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:26:22.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[80002]: cluster 2023-10-09T06:26:20.755110+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[80002]: cluster 2023-10-09T06:26:20.759195+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:22 smithi125 bash[80002]: cluster 2023-10-09T06:26:20.953013+0000 mgr.y (mgr.25723) 639 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:26:22.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:22.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:22 smithi155 bash[59495]: cluster 2023-10-09T06:26:20.755110+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:22 smithi155 bash[59495]: cluster 2023-10-09T06:26:20.759195+0000 osd.4 (osd.4) 80 : cluster 3 failed to encode map e154 with expected crc 2023-10-09T06:26:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:22 smithi155 bash[59495]: cluster 2023-10-09T06:26:20.953013+0000 mgr.y (mgr.25723) 639 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:26:22.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:22.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:22.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:22.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:22.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:22.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:22.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:22.817 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:22.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:22.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:22.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:23.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:23.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:23.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:23.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:23.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:23.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:23.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:23.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:23.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:23.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:23.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:23.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:23.639 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:23.646 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:23.646 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:23.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:23.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:23.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:23.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:23.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:23.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:24.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:24.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:24.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:24.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:24.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:24.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:24 smithi125 bash[77299]: cluster 2023-10-09T06:26:22.953734+0000 mgr.y (mgr.25723) 640 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:26:24.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:24 smithi125 bash[77299]: audit 2023-10-09T06:26:23.433449+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:24.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:24 smithi125 bash[80002]: cluster 2023-10-09T06:26:22.953734+0000 mgr.y (mgr.25723) 640 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:26:24.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:24 smithi125 bash[80002]: audit 2023-10-09T06:26:23.433449+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:24.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:24.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:24 smithi155 bash[59495]: cluster 2023-10-09T06:26:22.953734+0000 mgr.y (mgr.25723) 640 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:26:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:24 smithi155 bash[59495]: audit 2023-10-09T06:26:23.433449+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:24.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:24.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:24.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:24.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:24.691 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:24.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:24.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:24.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:24.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:24.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:24.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:24.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:25.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:25.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:25.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:26:25.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:25.271 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:25.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:25.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:25.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:25.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:25.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:25.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:26:25.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:25.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:25.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:25.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:25.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:25.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:26.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:26.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:26 smithi125 bash[77299]: cluster 2023-10-09T06:26:24.954259+0000 mgr.y (mgr.25723) 641 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:26.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:26 smithi125 bash[80002]: cluster 2023-10-09T06:26:24.954259+0000 mgr.y (mgr.25723) 641 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:26.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:26.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:26 smithi155 bash[59495]: cluster 2023-10-09T06:26:24.954259+0000 mgr.y (mgr.25723) 641 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:26.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:26.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:26.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:26.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:26.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:26.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:26.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:26.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:27.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:27.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:27.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:27.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:27.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:27.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:27.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:27.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:27.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:27.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:27.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:27.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:27.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:27.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:28.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:28.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:28 smithi125 bash[77299]: cluster 2023-10-09T06:26:26.954780+0000 mgr.y (mgr.25723) 642 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T06:26:28.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:28 smithi125 bash[80002]: cluster 2023-10-09T06:26:26.954780+0000 mgr.y (mgr.25723) 642 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T06:26:28.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:28.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:28.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:28.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:28.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:28.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:28 smithi155 bash[59495]: cluster 2023-10-09T06:26:26.954780+0000 mgr.y (mgr.25723) 642 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-09T06:26:28.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:28.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:28.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:28.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:28.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:28.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:28.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:29.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:29.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:29.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:29.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:29.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:29.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:29.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:29.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:29.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:29.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:29 smithi125 bash[77299]: cluster 2023-10-09T06:26:28.955252+0000 mgr.y (mgr.25723) 643 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:29.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:29 smithi125 bash[80002]: cluster 2023-10-09T06:26:28.955252+0000 mgr.y (mgr.25723) 643 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:29.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.960 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:29.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:30.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:30.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:30.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:29 smithi155 bash[59495]: cluster 2023-10-09T06:26:28.955252+0000 mgr.y (mgr.25723) 643 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:26:30.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:30.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:30.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:30.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:30.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:30.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:30.642 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:30.744 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:30.876 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:30.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:31.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:31.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:31.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:29.898547+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:29.911837+0000 mon.a (mon.0) 836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:30.161043+0000 mon.a (mon.0) 837 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:30.172833+0000 mon.a (mon.0) 838 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:30.764379+0000 mon.a (mon.0) 839 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:30.768118+0000 mon.a (mon.0) 840 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:26:31.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:30 smithi155 bash[59495]: audit 2023-10-09T06:26:30.776428+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:29.898547+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:29.911837+0000 mon.a (mon.0) 836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:30.161043+0000 mon.a (mon.0) 837 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:30.172833+0000 mon.a (mon.0) 838 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:30.764379+0000 mon.a (mon.0) 839 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:26:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:30.768118+0000 mon.a (mon.0) 840 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[77299]: audit 2023-10-09T06:26:30.776428+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:29.898547+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:29.911837+0000 mon.a (mon.0) 836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:30.161043+0000 mon.a (mon.0) 837 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:30.172833+0000 mon.a (mon.0) 838 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:30.764379+0000 mon.a (mon.0) 839 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:30.768118+0000 mon.a (mon.0) 840 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:26:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:30 smithi125 bash[80002]: audit 2023-10-09T06:26:30.776428+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:31.183 INFO:teuthology.orchestra.run.smithi125.stdout:moved 14 / 267 (5.24345%) 2023-10-09T06:26:31.183 INFO:teuthology.orchestra.run.smithi125.stdout:avg 33.375 2023-10-09T06:26:31.183 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 5.09749 -> 5.93585 (expected baseline 5.40399) 2023-10-09T06:26:31.183 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.5 with 26 -> 28 pgs (0.779026 -> 0.838951 * mean) 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.3 with 41 -> 31 pgs (1.22846 -> 0.928839 * mean) 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:oload 110 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 1 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0140 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.0154 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3 weight 1.0000 -> 0.6464 2023-10-09T06:26:31.184 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2 weight 1.0000 -> 0.7726 2023-10-09T06:26:31.194 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:26:31.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:31.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:31.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:31.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:31.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:31.482 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[73481]: debug 2023-10-09T06:26:31.167+0000 7f8497396700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-09T06:26:31.482 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:31] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:26:31.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:31.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:31.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:31.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:31.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:31.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:31.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:31.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:31.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:31 smithi155 bash[59495]: cluster 2023-10-09T06:26:30.957886+0000 mgr.y (mgr.25723) 644 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:31.964 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:31 smithi155 bash[59495]: audit 2023-10-09T06:26:31.173608+0000 mgr.y (mgr.25723) 645 : audit 0 from='client.35142 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:31.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:31.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:32.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:32.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[77299]: cluster 2023-10-09T06:26:30.957886+0000 mgr.y (mgr.25723) 644 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:32.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[77299]: audit 2023-10-09T06:26:31.173608+0000 mgr.y (mgr.25723) 645 : audit 0 from='client.35142 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:32.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[80002]: cluster 2023-10-09T06:26:30.957886+0000 mgr.y (mgr.25723) 644 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:32.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:31 smithi125 bash[80002]: audit 2023-10-09T06:26:31.173608+0000 mgr.y (mgr.25723) 645 : audit 0 from='client.35142 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:32.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:32.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:32.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:32.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:32.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:32.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:32.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:32.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:32.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:32.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:32.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:33.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:33.206 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:33.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:33.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:33.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:33.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:33.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:33.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:33.826 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:33.826 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:33.882 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:26:33.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T06:26:33.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:33.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:34.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:34.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:34 smithi125 bash[77299]: cluster 2023-10-09T06:26:32.958624+0000 mgr.y (mgr.25723) 646 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:34.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:34 smithi125 bash[77299]: audit 2023-10-09T06:26:33.578083+0000 mon.a (mon.0) 842 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:34 smithi125 bash[80002]: cluster 2023-10-09T06:26:32.958624+0000 mgr.y (mgr.25723) 646 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:34.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:34 smithi125 bash[80002]: audit 2023-10-09T06:26:33.578083+0000 mon.a (mon.0) 842 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:34.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:34.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:34.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:34.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:34.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:34.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:34 smithi155 bash[59495]: cluster 2023-10-09T06:26:32.958624+0000 mgr.y (mgr.25723) 646 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2023-10-09T06:26:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:34 smithi155 bash[59495]: audit 2023-10-09T06:26:33.578083+0000 mon.a (mon.0) 842 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:34.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:34.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:34.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:34.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:34.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:34.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:34.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:34.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:34.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:34.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:34.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:35.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:35.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:35.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:35.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:35.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:35.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:35.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:35.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:35.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:35.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:35.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:35.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:35.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:35.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:36.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:36.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:36.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:36.178 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:36 smithi125 bash[77299]: cluster 2023-10-09T06:26:34.960910+0000 mgr.y (mgr.25723) 647 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T06:26:36.178 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:36 smithi125 bash[80002]: cluster 2023-10-09T06:26:34.960910+0000 mgr.y (mgr.25723) 647 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T06:26:36.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:36.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:36.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:36 smithi155 bash[59495]: cluster 2023-10-09T06:26:34.960910+0000 mgr.y (mgr.25723) 647 : cluster 0 pgmap v621: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-09T06:26:36.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:36.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:36.558 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:26:36.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:36.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:36.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:26:36.669 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:26:36.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.722 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:26:36.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:36.758 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:26:36.795 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:26:36.841 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:26:36.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:36.875 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:26:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:36.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:36.889 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.890 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:26:36.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:37.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:37.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:37.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:37.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:37.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:37.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:37.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:37 smithi125 bash[77299]: audit 2023-10-09T06:26:36.503796+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.125:0/2337102859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:37.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:37 smithi125 bash[80002]: audit 2023-10-09T06:26:36.503796+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.125:0/2337102859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:37.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:37.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:37.390 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:37 smithi155 bash[59495]: audit 2023-10-09T06:26:36.503796+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.125:0/2337102859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:37.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:37.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:37.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:37.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:37.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:37.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:37.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:37.804 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:37.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:37.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:37.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:37.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:38.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:38.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:38 smithi125 bash[77299]: cluster 2023-10-09T06:26:36.961368+0000 mgr.y (mgr.25723) 648 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:26:38.042 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:38 smithi125 bash[80002]: cluster 2023-10-09T06:26:36.961368+0000 mgr.y (mgr.25723) 648 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:26:38.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:38.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:38.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:38.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:38 smithi155 bash[59495]: cluster 2023-10-09T06:26:36.961368+0000 mgr.y (mgr.25723) 648 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:26:38.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:38.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:38.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:38.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:38.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:38.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:38.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:39.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:39.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:39.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:39.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:39.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:39.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:39.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:39.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:39.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:39.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:39.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:39.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:39.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:39.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:39.834 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:39.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:39.960 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:39 smithi125 bash[77299]: cluster 2023-10-09T06:26:38.961847+0000 mgr.y (mgr.25723) 649 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:26:39.960 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:39 smithi125 bash[80002]: cluster 2023-10-09T06:26:38.961847+0000 mgr.y (mgr.25723) 649 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:26:39.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:39.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:40.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:40.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:40.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:39 smithi155 bash[59495]: cluster 2023-10-09T06:26:38.961847+0000 mgr.y (mgr.25723) 649 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:26:40.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:40.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:40.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:40.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:40.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:40.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:40.598 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:40.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:40.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:40.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:40.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:40.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:40.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:40.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:41.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:41.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:41.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:41.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:41.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:41.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:41.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:41] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-10-09T06:26:41.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:41.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:41.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:41.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:41.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:41.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:41.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:41.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:41.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:41.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:41.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:26:42.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:42.043 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:42.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:42.049 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:42 smithi125 bash[77299]: cluster 2023-10-09T06:26:40.962429+0000 mgr.y (mgr.25723) 650 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:26:42.049 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:42 smithi125 bash[80002]: cluster 2023-10-09T06:26:40.962429+0000 mgr.y (mgr.25723) 650 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:26:42.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:42.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:42.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:42 smithi155 bash[59495]: cluster 2023-10-09T06:26:40.962429+0000 mgr.y (mgr.25723) 650 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:26:42.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:42.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:42.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:42.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:42.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:42.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:42.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:42.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:42.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:43.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:43.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:43.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:43.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:43.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:43.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:43.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:43.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:43.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:43.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:43.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:43.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:43.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:43.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:43.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:43.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:43.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:43.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:44.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:44.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:44.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:44 smithi125 bash[77299]: cluster 2023-10-09T06:26:42.962855+0000 mgr.y (mgr.25723) 651 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:44.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:44 smithi125 bash[80002]: cluster 2023-10-09T06:26:42.962855+0000 mgr.y (mgr.25723) 651 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:44.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:44 smithi155 bash[59495]: cluster 2023-10-09T06:26:42.962855+0000 mgr.y (mgr.25723) 651 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15907 5.b 5:d9340d3e:::benchmark_data_smithi125_106927_object15906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.149 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.789447+0000", 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.346316145, 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33442039099999998, 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15907 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.150 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789447+0000", 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789447+0000", 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.151 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789447+0000", 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789468+0000", 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1665999999999999e-05 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.152 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789475+0000", 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6529999999999997e-06 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789526+0000", 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0813999999999998e-05 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.153 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789578+0000", 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1845999999999998e-05 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.789757+0000", 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017910800000000001 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.154 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.792829+0000", 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003072122 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.793962+0000", 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001133057 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.155 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123610+0000", 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.329648196 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123783+0000", 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017276699999999999 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123830+0000", 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6965000000000002e-05 2023-10-09T06:26:44.156 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123868+0000", 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7787999999999998e-05 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15910 5.4 5:22ea77b1:::benchmark_data_smithi125_106927_object15909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.791483+0000", 2023-10-09T06:26:44.157 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34428004300003, 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33363734099999998, 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15910 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.158 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791483+0000", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791483+0000", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791483+0000", 2023-10-09T06:26:44.159 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791506+0000", 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3377e-05 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791514+0000", 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8310000000000001e-06 2023-10-09T06:26:44.160 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791519+0000", 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6969999999999998e-06 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.792767+0000", 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001247677 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.161 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.792882+0000", 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011498299999999999 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.793013+0000", 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000131237 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.162 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.793936+0000", 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00092316299999999996 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123522+0000", 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.329585919 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125017+0000", 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001494135 2023-10-09T06:26:44.163 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125078+0000", 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1072000000000001e-05 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125121+0000", 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3105999999999997e-05 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.164 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15912 5.6 5:6d724260:::benchmark_data_smithi125_106927_object15911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.791534+0000", 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34422958200003, 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33214874799999999, 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.165 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15912 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791534+0000", 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.166 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791534+0000", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791533+0000", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791555+0000", 2023-10-09T06:26:44.167 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1865000000000002e-05 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791562+0000", 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7070000000000003e-06 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791567+0000", 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4299999999999997e-06 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791653+0000", 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5970000000000005e-05 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791804+0000", 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0001506 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.169 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.791957+0000", 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015351199999999999 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.793747+0000", 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017900119999999999 2023-10-09T06:26:44.170 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123491+0000", 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32974389700000001 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123583+0000", 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2155000000000001e-05 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.171 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123636+0000", 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2558000000000001e-05 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123683+0000", 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6588999999999997e-05 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.172 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15916 5.6 5:6d79d1f1:::benchmark_data_smithi125_106927_object15915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794087+0000", 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34167664799998, 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.332385604, 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.173 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15916 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794087+0000", 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.174 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794087+0000", 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794086+0000", 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794111+0000", 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5494999999999999e-05 2023-10-09T06:26:44.175 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794120+0000", 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4910000000000005e-06 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794126+0000", 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8299999999999998e-06 2023-10-09T06:26:44.176 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794170+0000", 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3291999999999998e-05 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794597+0000", 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042757400000000001 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.177 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794828+0000", 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023110599999999999 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123501+0000", 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.328672819 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.178 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123548+0000", 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6984000000000003e-05 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126289+0000", 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027412320000000001 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.179 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126435+0000", 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014597300000000001 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126472+0000", 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6949000000000001e-05 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.180 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15917 5.4 5:217c3f96:::benchmark_data_smithi125_106927_object15916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794139+0000", 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34162485299998, 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33117902700000001, 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15917 2023-10-09T06:26:44.181 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794139+0000", 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794139+0000", 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.182 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794138+0000", 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794159+0000", 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0784e-05 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.183 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794166+0000", 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1620000000000002e-06 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794172+0000", 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8250000000000001e-06 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.184 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794215+0000", 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3019000000000003e-05 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794329+0000", 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011401299999999999 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794530+0000", 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020085200000000001 2023-10-09T06:26:44.185 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796346+0000", 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018162110000000001 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125166+0000", 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32882017699999999 2023-10-09T06:26:44.186 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125240+0000", 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3894999999999996e-05 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125283+0000", 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2891e-05 2023-10-09T06:26:44.187 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125318+0000", 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4712999999999998e-05 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15918 5.4 5:2314d320:::benchmark_data_smithi125_106927_object15917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794183+0000", 2023-10-09T06:26:44.188 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34158058100002, 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33124916500000001, 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15918 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.189 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794183+0000", 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794183+0000", 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.190 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794182+0000", 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794205+0000", 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2269e-05 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.191 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794212+0000", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.99e-06 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794217+0000", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.356e-06 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795317+0000", 2023-10-09T06:26:44.192 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010996980000000001 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795439+0000", 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122156 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795561+0000", 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012191 2023-10-09T06:26:44.193 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796354+0000", 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00079301299999999996 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125202+0000", 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32884855899999998 2023-10-09T06:26:44.194 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125363+0000", 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016054900000000001 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125400+0000", 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6705000000000002e-05 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.195 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125432+0000", 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2400999999999997e-05 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15920 5.8 5:1d25a8f1:::benchmark_data_smithi125_106927_object15919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794223+0000", 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.341540619, 2023-10-09T06:26:44.196 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33108438000000001, 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15920 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.197 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794223+0000", 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794223+0000", 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794222+0000", 2023-10-09T06:26:44.198 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794244+0000", 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1984000000000002e-05 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794251+0000", 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.199 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794256+0000", 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.344e-06 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794289+0000", 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2532e-05 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.200 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794410+0000", 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121734 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794550+0000", 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013948300000000001 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.201 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796153+0000", 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016028959999999999 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.124662+0000", 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.328509516 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125100+0000", 2023-10-09T06:26:44.202 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000437699 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125211+0000", 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111446 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125307+0000", 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5829999999999996e-05 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.203 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15922 5.6 5:63d7e1dd:::benchmark_data_smithi125_106927_object15921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794446+0000", 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34131767600002, 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33216351500000002, 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.204 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15922 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794446+0000", 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.205 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794446+0000", 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794445+0000", 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794467+0000", 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1654e-05 2023-10-09T06:26:44.206 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794475+0000", 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2029999999999992e-06 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794482+0000", 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.596e-06 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.207 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795851+0000", 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0013692590000000001 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795975+0000", 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012463200000000001 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:26:44.208 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796124+0000", 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014820999999999999 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123507+0000", 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32738342599999998 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123748+0000", 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024098300000000001 2023-10-09T06:26:44.209 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126535+0000", 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002787234 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126575+0000", 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9288000000000001e-05 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.210 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126609+0000", 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4694999999999999e-05 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15923 5.a 5:53fa90b2:::benchmark_data_smithi125_106927_object15922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.794759+0000", 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.34100471400001, 2023-10-09T06:26:44.211 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33071457199999998, 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15923 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794759+0000", 2023-10-09T06:26:44.212 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794759+0000", 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794758+0000", 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:44.213 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794779+0000", 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1449000000000001e-05 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794789+0000", 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9170000000000001e-06 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.214 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794796+0000", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8220000000000001e-06 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.794836+0000", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9904999999999998e-05 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795084+0000", 2023-10-09T06:26:44.215 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024812499999999997 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.795242+0000", 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015813599999999999 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.124595+0000", 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32935263999999997 2023-10-09T06:26:44.216 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125297+0000", 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070186599999999999 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125324+0000", 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7585000000000001e-05 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.217 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125371+0000", 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6282000000000002e-05 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.125473+0000", 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000102694 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.218 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15925 5.a 5:5913c3ec:::benchmark_data_smithi125_106927_object15924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.796244+0000", 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.339518973, 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33064746699999997, 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.219 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15925 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796244+0000", 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796244+0000", 2023-10-09T06:26:44.220 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796243+0000", 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796301+0000", 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8003e-05 2023-10-09T06:26:44.221 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796320+0000", 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8382000000000001e-05 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796328+0000", 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5169999999999999e-06 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.222 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796388+0000", 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9444000000000002e-05 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.223 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.225 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123380+0000", 2023-10-09T06:26:44.225 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32699231099999998 2023-10-09T06:26:44.225 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.225 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.225 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123698+0000", 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000317499 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126601+0000", 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00290311 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.226 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126610+0000", 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0089999999999996e-06 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126809+0000", 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000198946 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126857+0000", 2023-10-09T06:26:44.227 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8705000000000002e-05 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.126892+0000", 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4535000000000001e-05 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.228 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15926 5.b 5:d87fac38:::benchmark_data_smithi125_106927_object15925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.796337+0000", 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.33942669300001, 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33270902899999999, 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15926 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.229 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796337+0000", 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796337+0000", 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.230 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796336+0000", 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796368+0000", 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2342999999999998e-05 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.231 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796376+0000", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2409999999999998e-06 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796388+0000", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2377e-05 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796537+0000", 2023-10-09T06:26:44.232 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000149295 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796648+0000", 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110308 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796828+0000", 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018002799999999999 2023-10-09T06:26:44.233 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123553+0000", 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32672510100000002 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.127392+0000", 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038397129999999998 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.234 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.128946+0000", 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015539110000000001 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.129003+0000", 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6960999999999999e-05 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.235 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.129046+0000", 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2404999999999998e-05 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15927 5.4 5:27085dda:::benchmark_data_smithi125_106927_object15926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.796402+0000", 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.33936112800001, 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33423555799999999, 2023-10-09T06:26:44.236 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15927 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796402+0000", 2023-10-09T06:26:44.237 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796402+0000", 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796402+0000", 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.238 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796426+0000", 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3986e-05 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796433+0000", 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0339999999999999e-06 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.239 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796438+0000", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6330000000000001e-06 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123647+0000", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32720818499999998 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.123834+0000", 2023-10-09T06:26:44.240 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018708200000000001 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.124070+0000", 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023607399999999999 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.129941+0000", 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058711060000000001 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.241 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130014+0000", 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3029999999999997e-05 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130513+0000", 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049948899999999999 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.242 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.243 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130582+0000", 2023-10-09T06:26:44.243 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8491000000000005e-05 2023-10-09T06:26:44.243 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.243 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.244 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.244 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130638+0000", 2023-10-09T06:26:44.244 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5998000000000002e-05 2023-10-09T06:26:44.244 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.244 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:15928 5.8 5:1372e0e5:::benchmark_data_smithi125_106927_object15927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:14.796451+0000", 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 509.339312162, 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33383622899999998, 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.245 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 15928 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796451+0000", 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.246 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796451+0000", 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796451+0000", 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.247 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796480+0000", 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9254000000000001e-05 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796487+0000", 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4680000000000003e-06 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.248 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796494+0000", 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0990000000000004e-06 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796836+0000", 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034244599999999999 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.249 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.796947+0000", 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011126 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:14.797071+0000", 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012378500000000001 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.124556+0000", 2023-10-09T06:26:44.250 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32748538700000002 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.129241+0000", 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0046841770000000003 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130189+0000", 2023-10-09T06:26:44.251 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00094872999999999995 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130244+0000", 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4951000000000003e-05 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:15.130287+0000", 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3223000000000003e-05 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.252 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76660 5.d 5:bafea43a:::benchmark_data_smithi125_106927_object76659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.773454+0000", 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.36230937300002, 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3479620219999999, 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:26:44.253 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76660 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773454+0000", 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.254 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773454+0000", 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773453+0000", 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.255 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773476+0000", 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2748999999999999e-05 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773556+0000", 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9772999999999997e-05 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.256 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.773564+0000", 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9959999999999993e-06 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.774559+0000", 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00099532000000000006 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.257 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.774690+0000", 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130273 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.774827+0000", 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000137732 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.258 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.780606+0000", 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005778643 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.788494+0000", 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078883459999999992 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804235+0000", 2023-10-09T06:26:44.259 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015741102 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804253+0000", 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7380999999999998e-05 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781496+0000", 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97724305099999997 2023-10-09T06:26:44.260 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781512+0000", 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5744999999999999e-05 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.121279+0000", 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.339767793 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.261 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.121416+0000", 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136649 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76666 5.6 5:6d580558:::benchmark_data_smithi125_106927_object76665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T06:26:44.262 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.779181+0000", 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.356581914, 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3221710579999999, 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76666 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.263 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779181+0000", 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779181+0000", 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.264 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779180+0000", 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779207+0000", 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6795999999999999e-05 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779220+0000", 2023-10-09T06:26:44.265 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3236e-05 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779228+0000", 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4549999999999995e-06 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779280+0000", 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1588999999999997e-05 2023-10-09T06:26:44.266 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779439+0000", 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015897 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.779583+0000", 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014447 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.267 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.786649+0000", 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0070651289999999999 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.806973+0000", 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.020324231000000002 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.268 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.806985+0000", 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2028000000000001e-05 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.782162+0000", 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97517712400000001 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.269 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.782175+0000", 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3298e-05 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.101240+0000", 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3190647870000001 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.270 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.101353+0000", 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011256000000000001 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76669 5.d 5:b6a0434b:::benchmark_data_smithi125_106927_object76668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.795739+0000", 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.340024873, 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.349237526, 2023-10-09T06:26:44.271 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76669 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795739+0000", 2023-10-09T06:26:44.272 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795739+0000", 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795737+0000", 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.273 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795768+0000", 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1850999999999997e-05 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795783+0000", 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4885999999999999e-05 2023-10-09T06:26:44.274 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.795791+0000", 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8869999999999992e-06 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804506+0000", 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0087145220000000006 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.275 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804514+0000", 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4710000000000007e-06 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781543+0000", 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97702849700000005 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.276 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781553+0000", 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9720000000000004e-06 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.125498+0000", 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.343945137 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.125715+0000", 2023-10-09T06:26:44.277 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000216667 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.125933+0000", 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021896200000000001 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.138915+0000", 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012981779000000001 2023-10-09T06:26:44.278 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.139039+0000", 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012342599999999999 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.141787+0000", 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027487369999999998 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.279 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.141881+0000", 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3289999999999996e-05 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.144976+0000", 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030953249999999999 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.280 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76670 5.d 5:b10f00fa:::benchmark_data_smithi125_106927_object76669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.796057+0000", 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.33970687200002, 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3707614929999998, 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.281 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76670 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796057+0000", 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.282 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796057+0000", 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796056+0000", 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.283 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796079+0000", 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3569000000000001e-05 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796102+0000", 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2847999999999999e-05 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.796113+0000", 2023-10-09T06:26:44.284 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1165000000000001e-05 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804537+0000", 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0084236480000000006 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.804543+0000", 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.122e-06 2023-10-09T06:26:44.285 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781579+0000", 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97703613700000003 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781587+0000", 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2239999999999995e-06 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.286 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.128794+0000", 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3472068990000001 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.129142+0000", 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000347301 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.287 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.129325+0000", 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018361500000000001 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.138957+0000", 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0096313560000000006 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.288 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.145174+0000", 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062175549999999996 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.163004+0000", 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017829491999999999 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.289 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.163095+0000", 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0977000000000007e-05 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.166818+0000", 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037234960000000002 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.290 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76671 5.d 5:b53e6b62:::benchmark_data_smithi125_106927_object76670:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.799008+0000", 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.336754918, 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3713857699999998, 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76671 2023-10-09T06:26:44.291 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799008+0000", 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799008+0000", 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.292 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799007+0000", 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799034+0000", 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7552e-05 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.293 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799057+0000", 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2745e-05 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.799065+0000", 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2949999999999994e-06 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.294 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.806466+0000", 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0074006159999999996 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.806479+0000", 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2556e-05 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781614+0000", 2023-10-09T06:26:44.295 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.97513551799999998 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781622+0000", 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8800000000000008e-06 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.132702+0000", 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3510797029999999 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.296 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.132819+0000", 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000117371 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.132946+0000", 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012730700000000001 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.297 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.138995+0000", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060486890000000003 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.145184+0000", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061885489999999998 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.166886+0000", 2023-10-09T06:26:44.298 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.021702125999999999 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.166973+0000", 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7292000000000004e-05 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.170394+0000", 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0034212069999999999 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.299 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76682 5.d 5:b7bfb727:::benchmark_data_smithi125_106927_object76681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.817119+0000", 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.31864408799999, 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.356628089, 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.300 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76682 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817119+0000", 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.301 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817119+0000", 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817119+0000", 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.302 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817137+0000", 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7893999999999998e-05 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817142+0000", 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8960000000000002e-06 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817145+0000", 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.625e-06 2023-10-09T06:26:44.303 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817262+0000", 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011626 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.817272+0000", 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0149000000000001e-05 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.304 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781756+0000", 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.96448464199999995 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.781774+0000", 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7657000000000001e-05 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.305 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.139094+0000", 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.357320276 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.139263+0000", 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016871599999999999 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.139375+0000", 2023-10-09T06:26:44.306 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011234 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.145194+0000", 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058181589999999998 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.145236+0000", 2023-10-09T06:26:44.307 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2382999999999997e-05 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.170467+0000", 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.02523129 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.170536+0000", 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9175000000000001e-05 2023-10-09T06:26:44.308 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.173747+0000", 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032109899999999999 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76690 5.6 5:678a4709:::benchmark_data_smithi125_106927_object76689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T06:26:44.309 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:18:58.830735+0000", 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 465.30502870700002, 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2873545430000002, 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76690 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:26:44.310 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830735+0000", 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830735+0000", 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:26:44.311 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830733+0000", 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830757+0000", 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4011000000000001e-05 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830769+0000", 2023-10-09T06:26:44.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1637999999999999e-05 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830776+0000", 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9619999999999996e-06 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830839+0000", 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2885999999999994e-05 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:58.830847+0000", 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8699999999999992e-06 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.782203+0000", 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.95135607200000005 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:18:59.782212+0000", 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1379999999999996e-06 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.105629+0000", 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3234171850000001 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:26:44.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.105802+0000", 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000172332 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.105964+0000", 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000162907 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.108877+0000", 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029121469999999999 2023-10-09T06:26:44.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.109248+0000", 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00037125199999999998 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.114304+0000", 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050565669999999997 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.114365+0000", 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.1017999999999998e-05 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.118089+0000", 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037238079999999999 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.318 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:26:44.319 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:26:44.319 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:26:44.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:44.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:44.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:44.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:44.591 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:44.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:44.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:44.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:44.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:44.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:44.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:44.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:45.057 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:26:45.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:45.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:45 smithi125 bash[77299]: audit 2023-10-09T06:26:44.257680+0000 mon.a (mon.0) 844 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:45.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:45 smithi125 bash[80002]: audit 2023-10-09T06:26:44.257680+0000 mon.a (mon.0) 844 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:45.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:45.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:45.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:45.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:45 smithi155 bash[59495]: audit 2023-10-09T06:26:44.257680+0000 mon.a (mon.0) 844 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:45.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:45.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:45.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:45.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:45.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:45.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:45.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:45.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:45.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:46.046 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:44.963446+0000 mgr.y (mgr.25723) 652 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.020035+0000 mon.a (mon.0) 845 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.020075+0000 mon.a (mon.0) 846 : cluster 1 Cluster is now healthy 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.043671+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: audit 2023-10-09T06:26:45.046439+0000 mon.a (mon.0) 847 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.046814+0000 mon.a (mon.0) 848 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.053944+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[77299]: cluster 2023-10-09T06:26:45.058349+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.047 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:44.963446+0000 mgr.y (mgr.25723) 652 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T06:26:46.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:46.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:44.963446+0000 mgr.y (mgr.25723) 652 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.020035+0000 mon.a (mon.0) 845 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.020075+0000 mon.a (mon.0) 846 : cluster 1 Cluster is now healthy 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.043671+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: audit 2023-10-09T06:26:45.046439+0000 mon.a (mon.0) 847 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.046814+0000 mon.a (mon.0) 848 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:26:46.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.053944+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:46 smithi155 bash[59495]: cluster 2023-10-09T06:26:45.058349+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:46.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:46.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:46.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.020035+0000 mon.a (mon.0) 845 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:46.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.020075+0000 mon.a (mon.0) 846 : cluster 1 Cluster is now healthy 2023-10-09T06:26:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.043671+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: audit 2023-10-09T06:26:45.046439+0000 mon.a (mon.0) 847 : audit 1 from='client.? 172.21.15.125:0/1320572702' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.046814+0000 mon.a (mon.0) 848 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:26:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.053944+0000 osd.4 (osd.4) 81 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:46 smithi125 bash[80002]: cluster 2023-10-09T06:26:45.058349+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-09T06:26:46.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.503 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:46.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:46.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:46.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:46.744 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:46.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:46.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:46.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:47.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:47.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:47.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:47.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:47.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:47.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:47.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:26:47.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:47.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:47.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:47.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:47.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:47.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:47.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:47.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:47.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:47.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:47.882 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:47.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:47.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:48.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:48.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:48.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:48 smithi155 bash[59495]: cluster 2023-10-09T06:26:46.963958+0000 mgr.y (mgr.25723) 653 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:48.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:48.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:48.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:48.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:48 smithi125 bash[77299]: cluster 2023-10-09T06:26:46.963958+0000 mgr.y (mgr.25723) 653 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:48.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:48 smithi125 bash[80002]: cluster 2023-10-09T06:26:46.963958+0000 mgr.y (mgr.25723) 653 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:48.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:48.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:48.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:48.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:48.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:48.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:48.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:48.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:48.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:48.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:48.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:48.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:48.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:49.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:49.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:49.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:49.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:49.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:49.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:49.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:49.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:49.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:49.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[77299]: audit 2023-10-09T06:26:48.585991+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:49.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[77299]: audit 2023-10-09T06:26:48.587608+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:49.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[77299]: cluster 2023-10-09T06:26:48.966902+0000 mgr.y (mgr.25723) 654 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:49.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[80002]: audit 2023-10-09T06:26:48.585991+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:49.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[80002]: audit 2023-10-09T06:26:48.587608+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:49.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:49 smithi125 bash[80002]: cluster 2023-10-09T06:26:48.966902+0000 mgr.y (mgr.25723) 654 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:avg 33.375 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 5.09749 -> 5.09749 (expected baseline 5.40399) 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.5 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:26:49.635 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 3 2023-10-09T06:26:49.636 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:26:49.636 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0109 2023-10-09T06:26:49.636 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.1091 2023-10-09T06:26:49.645 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:26:49.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:49.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:49.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:49 smithi155 bash[59495]: audit 2023-10-09T06:26:48.585991+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:26:49.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:49 smithi155 bash[59495]: audit 2023-10-09T06:26:48.587608+0000 mon.a (mon.0) 850 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:49.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:49 smithi155 bash[59495]: cluster 2023-10-09T06:26:48.966902+0000 mgr.y (mgr.25723) 654 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:49.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:49.880 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:49.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:49.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:49.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:26:50.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:50.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:50.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:50.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:50.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:50.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:50.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:50.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:50.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:50.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:50.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:50.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:50.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:50.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:50.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:50.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:50 smithi125 bash[77299]: audit 2023-10-09T06:26:49.629132+0000 mgr.y (mgr.25723) 655 : audit 0 from='client.35160 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:50.830 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:50 smithi125 bash[80002]: audit 2023-10-09T06:26:49.629132+0000 mgr.y (mgr.25723) 655 : audit 0 from='client.35160 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:50.841 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:26:50.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:26:50.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:50.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:50.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:50.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:51.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:51.109 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:50 smithi155 bash[59495]: audit 2023-10-09T06:26:49.629132+0000 mgr.y (mgr.25723) 655 : audit 0 from='client.35160 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:26:51.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:51.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:51.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:51.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:51.319 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:51.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:51.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:51.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:51.530 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:26:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:26:51] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:26:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:51.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:51.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:51.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:51.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:51.810 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:51 smithi125 bash[77299]: cluster 2023-10-09T06:26:50.967675+0000 mgr.y (mgr.25723) 656 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:51.810 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:51 smithi125 bash[80002]: cluster 2023-10-09T06:26:50.967675+0000 mgr.y (mgr.25723) 656 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:51.837 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:51.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:51.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:51.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:51.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:52.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:52.082 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:51 smithi155 bash[59495]: cluster 2023-10-09T06:26:50.967675+0000 mgr.y (mgr.25723) 656 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:26:52.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:52.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:52.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:52.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:52.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:52.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:52.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:52.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:52.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:52.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:52.742 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:52.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:52.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:52.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:52.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:53.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:53.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:53.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:53.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:53.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:53.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:53.360 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:53.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:53.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:53.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:53.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:53.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:53.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:53.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:53.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:53.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:53.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:53.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:54.029 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:54 smithi125 bash[77299]: cluster 2023-10-09T06:26:52.968288+0000 mgr.y (mgr.25723) 657 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:26:54.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:54.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:54 smithi155 bash[59495]: cluster 2023-10-09T06:26:52.968288+0000 mgr.y (mgr.25723) 657 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:26:54.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:54.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:54.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:54.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.366 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:54 smithi125 bash[80002]: cluster 2023-10-09T06:26:52.968288+0000 mgr.y (mgr.25723) 657 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:26:54.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:54.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:54.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:54.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:54.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:54.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:54.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:54.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:54.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:54.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:54.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:55.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:55.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:55.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:55.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:55.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:55.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:55.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:55.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:55.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:55.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:55.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:55.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:55.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:55.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:55.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:55.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:55.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:55.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:55.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:55.996 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:55.996 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:56.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:56.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:56.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:56.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:56.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.295 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:56 smithi155 bash[59495]: cluster 2023-10-09T06:26:54.968833+0000 mgr.y (mgr.25723) 658 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T06:26:56.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:56 smithi125 bash[77299]: cluster 2023-10-09T06:26:54.968833+0000 mgr.y (mgr.25723) 658 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T06:26:56.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:56 smithi125 bash[80002]: cluster 2023-10-09T06:26:54.968833+0000 mgr.y (mgr.25723) 658 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-09T06:26:56.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:56.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:56.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:56.609 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:56.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:56.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:56.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:56.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:57.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:57.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:57.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:57.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:57.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:57.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:57.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:57.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:57.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:57.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:57.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:57.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:57.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:57.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:57.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:57.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:58.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:58.122 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:58 smithi155 bash[59495]: cluster 2023-10-09T06:26:56.969573+0000 mgr.y (mgr.25723) 659 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:26:58.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:58.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:58.379 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:58.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:58 smithi125 bash[77299]: cluster 2023-10-09T06:26:56.969573+0000 mgr.y (mgr.25723) 659 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:26:58.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:58 smithi125 bash[80002]: cluster 2023-10-09T06:26:56.969573+0000 mgr.y (mgr.25723) 659 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:26:58.402 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:58.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:58.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:58.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:58.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:58.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:58.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:58.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:58.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:59.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:59.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:59.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:59.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:59.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:59.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:59.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:59.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:59.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:59.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:59.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:59.880 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:26:59 smithi125 bash[80002]: cluster 2023-10-09T06:26:58.970129+0000 mgr.y (mgr.25723) 660 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:26:59.881 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:26:59 smithi125 bash[77299]: cluster 2023-10-09T06:26:58.970129+0000 mgr.y (mgr.25723) 660 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:26:59.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:26:59.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:00.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:00.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:00.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:26:59 smithi155 bash[59495]: cluster 2023-10-09T06:26:58.970129+0000 mgr.y (mgr.25723) 660 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:27:00.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:00.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:00.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:00.310 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:27:00.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:00.357 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:27:00.413 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:27:00.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:00.457 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:00.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:00.465 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:27:00.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:27:00.546 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:27:00.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:00.579 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:27:00.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:00.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:00.619 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:27:00.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.623 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.624 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:27:00.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:00.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:00.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:00.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:00.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:00.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:00.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:01.002 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:00 smithi125 bash[77299]: audit 2023-10-09T06:27:00.285849+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.125:0/158552140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.002 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:00 smithi125 bash[80002]: audit 2023-10-09T06:27:00.285849+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.125:0/158552140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:01.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:01.142 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:00 smithi155 bash[59495]: audit 2023-10-09T06:27:00.285849+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.125:0/158552140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.230 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:01.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:01.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:01.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:01] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:27:01.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:01.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:01.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:01.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:01.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:01.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:01.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:01.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:01.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:01.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:02.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:02.061 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:01 smithi155 bash[59495]: cluster 2023-10-09T06:27:00.970812+0000 mgr.y (mgr.25723) 661 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:27:02.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:02.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:02.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:01 smithi125 bash[77299]: cluster 2023-10-09T06:27:00.970812+0000 mgr.y (mgr.25723) 661 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:27:02.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:01 smithi125 bash[80002]: cluster 2023-10-09T06:27:00.970812+0000 mgr.y (mgr.25723) 661 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:27:02.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:02.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:02.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:02.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:02.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:02.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:02.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:02.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:02.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:02.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:02.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:02.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:02.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:02.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:02.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:02.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:03.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:03.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:03.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:03.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:03.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:03.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:03.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:03.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:03.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:03.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:03.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:03.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:03.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:03.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:03.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:03.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:03.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:04.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:04.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:04.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:04.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:04.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:04 smithi155 bash[59495]: cluster 2023-10-09T06:27:02.971371+0000 mgr.y (mgr.25723) 662 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:27:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:04 smithi155 bash[59495]: audit 2023-10-09T06:27:03.580067+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:04.457 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:04 smithi125 bash[77299]: cluster 2023-10-09T06:27:02.971371+0000 mgr.y (mgr.25723) 662 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:27:04.457 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:04 smithi125 bash[77299]: audit 2023-10-09T06:27:03.580067+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:04.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:04 smithi125 bash[80002]: cluster 2023-10-09T06:27:02.971371+0000 mgr.y (mgr.25723) 662 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:27:04.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:04 smithi125 bash[80002]: audit 2023-10-09T06:27:03.580067+0000 mon.a (mon.0) 852 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:04.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:04.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:04.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:04.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:04.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:04.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:04.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:04.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:05.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:05.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:05.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:05.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:05.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:05.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:05.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:05.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:05.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:05.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:05.779 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:05.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:05.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:05.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:05.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:27:05.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:05.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:06.041 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:06 smithi155 bash[59495]: cluster 2023-10-09T06:27:04.972014+0000 mgr.y (mgr.25723) 663 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:27:06.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:06.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:06.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:06 smithi125 bash[77299]: cluster 2023-10-09T06:27:04.972014+0000 mgr.y (mgr.25723) 663 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:27:06.155 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:06 smithi125 bash[80002]: cluster 2023-10-09T06:27:04.972014+0000 mgr.y (mgr.25723) 663 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T06:27:06.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:06.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:06.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:06.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:06.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:06.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:06.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:06.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:06.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:06.744 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:06.744 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:06.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:06.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:06.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:06.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:06.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:07.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:07.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:07.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:07.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:07.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:07.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:07.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:07.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:07.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:07.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:07.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:07.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:07.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:07.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:08.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:08.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:08.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:08.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:08.305 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:08 smithi155 bash[59495]: cluster 2023-10-09T06:27:06.972439+0000 mgr.y (mgr.25723) 664 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:27:08.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:08 smithi125 bash[77299]: cluster 2023-10-09T06:27:06.972439+0000 mgr.y (mgr.25723) 664 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:27:08.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:08 smithi125 bash[80002]: cluster 2023-10-09T06:27:06.972439+0000 mgr.y (mgr.25723) 664 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:27:08.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:08.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:08.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:08.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:08.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:08.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:08.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:08.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:08.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:08.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:09.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:09.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:09.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:09.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:09.453 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:09.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:09.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.867 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:09 smithi125 bash[77299]: cluster 2023-10-09T06:27:08.972894+0000 mgr.y (mgr.25723) 665 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:27:09.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:10.681 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:09 smithi155 bash[59495]: cluster 2023-10-09T06:27:08.972894+0000 mgr.y (mgr.25723) 665 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:27:10.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:10.682 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:09 smithi125 bash[80002]: cluster 2023-10-09T06:27:08.972894+0000 mgr.y (mgr.25723) 665 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:27:10.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:27:11.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:11.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:11.334 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout:avg 33.375 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 5.09749 -> 5.09749 (expected baseline 5.40399) 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.5 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:27:11.335 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:27:11.338 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 1 2023-10-09T06:27:11.338 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:27:11.340 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0071 2023-10-09T06:27:11.340 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.0710 2023-10-09T06:27:11.342 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:27:11.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:11.360 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:11] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:27:11.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:11.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:11.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:11.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:12.049 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:27:12.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:12.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:12.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:12.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:12.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:12 smithi155 bash[59495]: cluster 2023-10-09T06:27:10.973432+0000 mgr.y (mgr.25723) 666 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 955 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:27:12.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:12 smithi155 bash[59495]: audit 2023-10-09T06:27:11.236656+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:12.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:12.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:12 smithi125 bash[77299]: cluster 2023-10-09T06:27:10.973432+0000 mgr.y (mgr.25723) 666 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 955 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:27:12.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:12 smithi125 bash[77299]: audit 2023-10-09T06:27:11.236656+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:12.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:12 smithi125 bash[80002]: cluster 2023-10-09T06:27:10.973432+0000 mgr.y (mgr.25723) 666 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 955 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:27:12.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:12 smithi125 bash[80002]: audit 2023-10-09T06:27:11.236656+0000 mon.a (mon.0) 853 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:12.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:12.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:12.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:12.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:12.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:12.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:12.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:12.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:13.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: audit 2023-10-09T06:27:11.321870+0000 mgr.y (mgr.25723) 667 : audit 0 from='client.45068 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.026946+0000 mon.a (mon.0) 854 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: audit 2023-10-09T06:27:12.036084+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.036440+0000 mon.a (mon.0) 856 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.036906+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.063279+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.064130+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:13 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.064872+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:13.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:13.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:13.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:13.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: audit 2023-10-09T06:27:11.321870+0000 mgr.y (mgr.25723) 667 : audit 0 from='client.45068 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:27:13.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.026946+0000 mon.a (mon.0) 854 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:13.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: audit 2023-10-09T06:27:12.036084+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:13.326 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.036440+0000 mon.a (mon.0) 856 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.036906+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.063279+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.064130+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.064872+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: audit 2023-10-09T06:27:11.321870+0000 mgr.y (mgr.25723) 667 : audit 0 from='client.45068 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.026946+0000 mon.a (mon.0) 854 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: audit 2023-10-09T06:27:12.036084+0000 mon.a (mon.0) 855 : audit 1 from='client.? 172.21.15.125:0/1711345869' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:13.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.036440+0000 mon.a (mon.0) 856 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:27:13.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.036906+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.063279+0000 osd.4 (osd.4) 82 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.064130+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.328 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:13 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.064872+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e156 with expected crc 2023-10-09T06:27:13.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:13.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:13.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:13.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:13.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:13.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:13.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:13.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.015 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:14.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:14.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:14.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:14 smithi155 bash[59495]: cluster 2023-10-09T06:27:12.973765+0000 mgr.y (mgr.25723) 668 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 879 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:27:14.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:14.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:14 smithi125 bash[77299]: cluster 2023-10-09T06:27:12.973765+0000 mgr.y (mgr.25723) 668 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 879 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:27:14.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:14 smithi125 bash[80002]: cluster 2023-10-09T06:27:12.973765+0000 mgr.y (mgr.25723) 668 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 879 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:27:14.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:14.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:14.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:14.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:14.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:14.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:14.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:14.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:14.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:15.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:15.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:15.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:15.067 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:15.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:15.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:15.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:15.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:15.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:15.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:15.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:15.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:27:15.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:15.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:15.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:15.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:15.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:15.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:15.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:15.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:16.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:16.105 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:16 smithi155 bash[59495]: cluster 2023-10-09T06:27:14.974416+0000 mgr.y (mgr.25723) 669 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 738 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:27:16.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:16.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:16.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:16.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:16.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:16.377 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:16 smithi125 bash[80002]: cluster 2023-10-09T06:27:14.974416+0000 mgr.y (mgr.25723) 669 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 738 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:27:16.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:16 smithi125 bash[77299]: cluster 2023-10-09T06:27:14.974416+0000 mgr.y (mgr.25723) 669 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 738 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:27:16.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:16.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:16.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:16.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:16.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:16.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:16.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:16.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:16.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:16.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:16.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:17.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:17.105 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:17.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:17.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:17.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:17.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:17.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:17.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:17.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:17.724 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:17.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:17.792 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:17.792 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:17.792 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T06:27:17.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:27:17.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:17.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:17.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:18.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:18.077 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:18 smithi125 bash[77299]: cluster 2023-10-09T06:27:16.974967+0000 mgr.y (mgr.25723) 670 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 669 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:27:18.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:18.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:18.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:18 smithi155 bash[59495]: cluster 2023-10-09T06:27:16.974967+0000 mgr.y (mgr.25723) 670 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 669 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:27:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:18.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:18.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:18 smithi125 bash[80002]: cluster 2023-10-09T06:27:16.974967+0000 mgr.y (mgr.25723) 670 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 669 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:27:18.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:18.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:18.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:18.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:18.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:18.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:18.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:18.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:19.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:19.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:19.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:19.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:19.392 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:19.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:19.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:19.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:19.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:19.812 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:19 smithi155 bash[59495]: audit 2023-10-09T06:27:18.587714+0000 mon.a (mon.0) 857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:19 smithi155 bash[59495]: audit 2023-10-09T06:27:18.593489+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:19.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:19 smithi155 bash[59495]: cluster 2023-10-09T06:27:18.975486+0000 mgr.y (mgr.25723) 671 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 629 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:27:19.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:19.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[77299]: audit 2023-10-09T06:27:18.587714+0000 mon.a (mon.0) 857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:19.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[77299]: audit 2023-10-09T06:27:18.593489+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:19.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[77299]: cluster 2023-10-09T06:27:18.975486+0000 mgr.y (mgr.25723) 671 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 629 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:27:19.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[80002]: audit 2023-10-09T06:27:18.587714+0000 mon.a (mon.0) 857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:19.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[80002]: audit 2023-10-09T06:27:18.593489+0000 mon.a (mon.0) 858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:19.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:19 smithi125 bash[80002]: cluster 2023-10-09T06:27:18.975486+0000 mgr.y (mgr.25723) 671 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 629 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-09T06:27:19.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:19.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:20.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:20.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:20.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:20.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:20.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:20.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:20.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:20.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:20.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:20.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:20.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:20.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:21.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:21.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:21.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:21.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:21.176 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:21.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:21.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:21.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:21.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:21.429 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:21] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:27:21.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:21.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:21.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:21.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:21.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:21.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:21.783 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:27:21.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:21.819 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:27:21.843 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:27:21.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:21.879 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:27:21.921 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:27:21.952 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:27:21.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:21.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:21.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:21.989 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:27:22.026 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:27:22.031 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.033 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:27:22.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:22.098 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:22 smithi155 bash[59495]: cluster 2023-10-09T06:27:20.976064+0000 mgr.y (mgr.25723) 672 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 518 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:27:22.098 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:22 smithi155 bash[59495]: audit 2023-10-09T06:27:21.714651+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.125:0/399365237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:22.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:22.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:22.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:22.268 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:22 smithi125 bash[77299]: cluster 2023-10-09T06:27:20.976064+0000 mgr.y (mgr.25723) 672 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 518 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:27:22.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:22 smithi125 bash[77299]: audit 2023-10-09T06:27:21.714651+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.125:0/399365237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:22 smithi125 bash[80002]: cluster 2023-10-09T06:27:20.976064+0000 mgr.y (mgr.25723) 672 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 518 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:27:22.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:22 smithi125 bash[80002]: audit 2023-10-09T06:27:21.714651+0000 mon.a (mon.0) 859 : audit 0 from='client.? 172.21.15.125:0/399365237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:22.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:22.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:22.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:22.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:22.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:22.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:22.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:23.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:23.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:23.161 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:23.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:23.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:23.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:23.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:23.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:23.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:23.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:23.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:23.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:23.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:23.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:23.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:23.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:23.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:23.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:24.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:24.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:24.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:24 smithi125 bash[77299]: cluster 2023-10-09T06:27:22.976611+0000 mgr.y (mgr.25723) 673 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 431 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:24.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:24 smithi125 bash[80002]: cluster 2023-10-09T06:27:22.976611+0000 mgr.y (mgr.25723) 673 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 431 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:24.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:24.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:24.243 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:24 smithi155 bash[59495]: cluster 2023-10-09T06:27:22.976611+0000 mgr.y (mgr.25723) 673 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 431 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:24.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:24.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:24.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:24.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:24.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:24.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:24.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:24.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:24.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:24.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:24.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:24.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:25.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:25.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:25.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:25.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:25.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:25.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:25.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:25.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:25.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:25.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:25.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:25.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:25.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:25.983 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:26.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:26.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:26 smithi155 bash[59495]: cluster 2023-10-09T06:27:24.977242+0000 mgr.y (mgr.25723) 674 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:27:26.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:26.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:26.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:26.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:26.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:26 smithi125 bash[77299]: cluster 2023-10-09T06:27:24.977242+0000 mgr.y (mgr.25723) 674 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:27:26.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:26 smithi125 bash[80002]: cluster 2023-10-09T06:27:24.977242+0000 mgr.y (mgr.25723) 674 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 296 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s 2023-10-09T06:27:26.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:26.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:26.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:26.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:26.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:26.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:26.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:26.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:26.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:26.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:26.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:27.079 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:27.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:27.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:27.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:27.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:27.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:27.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:27.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:27.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:27:27.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:27.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:27.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:27.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:27.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:27.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:27.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:27.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:27.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:27.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:28.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:28.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:28 smithi125 bash[77299]: cluster 2023-10-09T06:27:26.977718+0000 mgr.y (mgr.25723) 675 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 227 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:27:28.054 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:28 smithi125 bash[80002]: cluster 2023-10-09T06:27:26.977718+0000 mgr.y (mgr.25723) 675 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 227 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:27:28.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:28.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:28.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:28.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:28.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:28.360 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:28.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:28.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:28 smithi155 bash[59495]: cluster 2023-10-09T06:27:26.977718+0000 mgr.y (mgr.25723) 675 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 227 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:27:28.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:28.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:28.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:28.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:28.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:28.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:28.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:28.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:28.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:28.872 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:27:28.872 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:27:28.872 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:27:28.872 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:27:28.872 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:27:28.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:28.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:28.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:29.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:29.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:29.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:29.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:29.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:29.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:29.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:29.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:29.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:29.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:29.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.952 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:29 smithi155 bash[59495]: cluster 2023-10-09T06:27:28.978243+0000 mgr.y (mgr.25723) 676 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 185 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:27:29.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:29.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:29.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:30.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:30.091 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:29 smithi125 bash[77299]: cluster 2023-10-09T06:27:28.978243+0000 mgr.y (mgr.25723) 676 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 185 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:27:30.091 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:29 smithi125 bash[80002]: cluster 2023-10-09T06:27:28.978243+0000 mgr.y (mgr.25723) 676 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 185 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:27:30.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:30.152 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 111188 objects 2023-10-09T06:27:30.152 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :155.959 2023-10-09T06:27:30.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:30.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:30.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:30.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:30.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:30.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:30.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:30.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:30.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:30.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:30.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:30.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:30.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:30.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:30.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:31.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:31.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:31.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:31.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:30 smithi125 bash[77299]: audit 2023-10-09T06:27:30.862688+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:27:31.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:30 smithi125 bash[80002]: audit 2023-10-09T06:27:30.862688+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:27:31.311 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:30 smithi155 bash[59495]: audit 2023-10-09T06:27:30.862688+0000 mon.a (mon.0) 860 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:27:31.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:31.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:31.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.575 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:31] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:27:31.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:31.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:31.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:31.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:27:31.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:31.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:31.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:31.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:32.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:32.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:31 smithi125 bash[77299]: cluster 2023-10-09T06:27:30.979183+0000 mgr.y (mgr.25723) 677 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:27:32.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:31 smithi125 bash[80002]: cluster 2023-10-09T06:27:30.979183+0000 mgr.y (mgr.25723) 677 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:27:32.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:32.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:32.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:31 smithi155 bash[59495]: cluster 2023-10-09T06:27:30.979183+0000 mgr.y (mgr.25723) 677 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 78 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:27:32.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:32.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:32.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:32.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:32.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:32.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:32.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:32.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:32.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:32.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:32.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:32.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:33.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:33.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:33.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:33.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:33.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:33.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:33.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:33.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:33.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:33.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:33.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:33.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:33.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:33.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:33.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:33.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:34.051 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:27:34.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:34.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:34.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:34.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:34.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:34 smithi155 bash[59495]: cluster 2023-10-09T06:27:32.979763+0000 mgr.y (mgr.25723) 678 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:34.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:34 smithi155 bash[59495]: audit 2023-10-09T06:27:33.432497+0000 mon.a (mon.0) 861 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:34.240 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:34 smithi155 bash[59495]: audit 2023-10-09T06:27:33.580156+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:34.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:34.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[77299]: cluster 2023-10-09T06:27:32.979763+0000 mgr.y (mgr.25723) 678 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:34.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[77299]: audit 2023-10-09T06:27:33.432497+0000 mon.a (mon.0) 861 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:34.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[77299]: audit 2023-10-09T06:27:33.580156+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:34.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[80002]: cluster 2023-10-09T06:27:32.979763+0000 mgr.y (mgr.25723) 678 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 26 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:27:34.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[80002]: audit 2023-10-09T06:27:33.432497+0000 mon.a (mon.0) 861 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:34.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:34 smithi125 bash[80002]: audit 2023-10-09T06:27:33.580156+0000 mon.a (mon.0) 862 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:34.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:34.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:34.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:34.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:34.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:34.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:34.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:34.778 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:27:34.780 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:27:34.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:34.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:34.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:34.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:34.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:35.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:35.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:35.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:35.164 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[77299]: cluster 2023-10-09T06:27:34.032864+0000 mon.a (mon.0) 863 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[77299]: audit 2023-10-09T06:27:34.041180+0000 mon.a (mon.0) 864 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[77299]: cluster 2023-10-09T06:27:34.041248+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[77299]: cluster 2023-10-09T06:27:34.041281+0000 mon.a (mon.0) 865 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[80002]: cluster 2023-10-09T06:27:34.032864+0000 mon.a (mon.0) 863 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[80002]: audit 2023-10-09T06:27:34.041180+0000 mon.a (mon.0) 864 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[80002]: cluster 2023-10-09T06:27:34.041248+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:35.165 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:35 smithi125 bash[80002]: cluster 2023-10-09T06:27:34.041281+0000 mon.a (mon.0) 865 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:27:35.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:35.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:35.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:35 smithi155 bash[59495]: cluster 2023-10-09T06:27:34.032864+0000 mon.a (mon.0) 863 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:35.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:35 smithi155 bash[59495]: audit 2023-10-09T06:27:34.041180+0000 mon.a (mon.0) 864 : audit 1 from='client.? 172.21.15.125:0/3506085120' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:35.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:35 smithi155 bash[59495]: cluster 2023-10-09T06:27:34.041248+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:35.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:35 smithi155 bash[59495]: cluster 2023-10-09T06:27:34.041281+0000 mon.a (mon.0) 865 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:27:35.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:35.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:35.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.580 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:35.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:35.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:35.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:35.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:35.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:35.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:35.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:36.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:36.088 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[77299]: audit 2023-10-09T06:27:34.752279+0000 mgr.y (mgr.25723) 679 : audit 0 from='client.35196 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:36.088 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[77299]: cluster 2023-10-09T06:27:34.980326+0000 mgr.y (mgr.25723) 680 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 466 op/s 2023-10-09T06:27:36.088 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[77299]: cluster 2023-10-09T06:27:35.310661+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.088 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[77299]: cluster 2023-10-09T06:27:35.580750+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.089 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[80002]: audit 2023-10-09T06:27:34.752279+0000 mgr.y (mgr.25723) 679 : audit 0 from='client.35196 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:36.089 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[80002]: cluster 2023-10-09T06:27:34.980326+0000 mgr.y (mgr.25723) 680 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 466 op/s 2023-10-09T06:27:36.089 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[80002]: cluster 2023-10-09T06:27:35.310661+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.089 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:36 smithi125 bash[80002]: cluster 2023-10-09T06:27:35.580750+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.090 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":653,"stamp":"2023-10-09T06:27:34.035835+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":16455627,"num_objects":207,"num_object_clones":0,"num_object_copies":621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":207,"num_whiteouts":0,"num_read":6290,"num_read_kb":13749,"num_write":223305,"num_write_kb":7131192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":1190420,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140869,"ondisk_log_size":140869,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2349524,"kb_used_data":87316,"kb_used_omap":30,"kb_used_meta":2262113,"kb_avail":747611692,"statfs":{"total":767960285184,"available":765554372608,"internally_reserved":0,"allocated":89411584,"data_stored":84598545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":2316403823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,5,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-435290112,"num_objects":-6642,"num_object_clones":0,"num_object_copies":-19926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6642,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.059591"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":137,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060703+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:26:45.060703+0000","last_peered":"2023-10-09T06:26:45.060703+0000","last_clean":"2023-10-09T06:26:45.060703+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:26:45.060703+0000","last_undegraded":"2023-10-09T06:26:45.060703+0000","last_fullsized":"2023-10-09T06:26:45.060703+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":129,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:27:11.628866+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:27:11.628866+0000","last_peered":"2023-10-09T06:27:11.628866+0000","last_clean":"2023-10-09T06:27:11.628866+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:27:11.628866+0000","last_undegraded":"2023-10-09T06:27:11.628866+0000","last_fullsized":"2023-10-09T06:27:11.628866+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.047232+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:26:45.047232+0000","last_peered":"2023-10-09T06:26:45.047232+0000","last_clean":"2023-10-09T06:26:45.047232+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:26:45.047232+0000","last_undegraded":"2023-10-09T06:26:45.047232+0000","last_fullsized":"2023-10-09T06:26:45.047232+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:12.048479+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:27:12.048479+0000","last_peered":"2023-10-09T06:27:12.048479+0000","last_clean":"2023-10-09T06:27:12.048479+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:27:12.048479+0000","last_undegraded":"2023-10-09T06:27:12.048479+0000","last_fullsized":"2023-10-09T06:27:12.048479+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:46.568890+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:26:46.568890+0000","last_peered":"2023-10-09T06:26:46.568890+0000","last_clean":"2023-10-09T06:26:46.568890+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:26:46.568890+0000","last_undegraded":"2023-10-09T06:26:46.568890+0000","last_fullsized":"2023-10-09T06:26:46.568890+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.053997+0000","last_change":"2023-10-09T06:13:43.440132+0000","last_active":"2023-10-09T06:26:45.053997+0000","last_peered":"2023-10-09T06:26:45.053997+0000","last_clean":"2023-10-09T06:26:45.053997+0000","last_became_active":"2023-10-09T06:13:43.439747+0000","last_became_peered":"2023-10-09T06:13:43.439747+0000","last_unstale":"2023-10-09T06:26:45.053997+0000","last_undegraded":"2023-10-09T06:26:45.053997+0000","last_fullsized":"2023-10-09T06:26:45.053997+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.179216+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:26:45.179216+0000","last_peered":"2023-10-09T06:26:45.179216+0000","last_clean":"2023-10-09T06:26:45.179216+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:26:45.179216+0000","last_undegraded":"2023-10-09T06:26:45.179216+0000","last_fullsized":"2023-10-09T06:26:45.179216+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.053709+0000","last_change":"2023-10-09T06:14:42.688101+0000","last_active":"2023-10-09T06:26:45.053709+0000","last_peered":"2023-10-09T06:26:45.053709+0000","last_clean":"2023-10-09T06:26:45.053709+0000","last_became_active":"2023-10-09T06:14:42.687766+0000","last_became_peered":"2023-10-09T06:14:42.687766+0000","last_unstale":"2023-10-09T06:26:45.053709+0000","last_undegraded":"2023-10-09T06:26:45.053709+0000","last_fullsized":"2023-10-09T06:26:45.053709+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:02.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.061002+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:26:45.061002+0000","last_peered":"2023-10-09T06:26:45.061002+0000","last_clean":"2023-10-09T06:26:45.061002+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:26:45.061002+0000","last_undegraded":"2023-10-09T06:26:45.061002+0000","last_fullsized":"2023-10-09T06:26:45.061002+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":137,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.051814+0000","last_change":"2023-10-09T06:18:59.801160+0000","last_active":"2023-10-09T06:26:45.051814+0000","last_peered":"2023-10-09T06:26:45.051814+0000","last_clean":"2023-10-09T06:26:45.051814+0000","last_became_active":"2023-10-09T06:18:59.800769+0000","last_became_peered":"2023-10-09T06:18:59.800769+0000","last_unstale":"2023-10-09T06:26:45.051814+0000","last_undegraded":"2023-10-09T06:26:45.051814+0000","last_fullsized":"2023-10-09T06:26:45.051814+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:43.960181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":54,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063418+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:26:45.063418+0000","last_peered":"2023-10-09T06:26:45.063418+0000","last_clean":"2023-10-09T06:26:45.063418+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:26:45.063418+0000","last_undegraded":"2023-10-09T06:26:45.063418+0000","last_fullsized":"2023-10-09T06:26:45.063418+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.047540+0000","last_change":"2023-10-09T06:14:12.890839+0000","last_active":"2023-10-09T06:26:45.047540+0000","last_peered":"2023-10-09T06:26:45.047540+0000","last_clean":"2023-10-09T06:26:45.047540+0000","last_became_active":"2023-10-09T06:14:12.890460+0000","last_became_peered":"2023-10-09T06:14:12.890460+0000","last_unstale":"2023-10-09T06:26:45.047540+0000","last_undegraded":"2023-10-09T06:26:45.047540+0000","last_fullsized":"2023-10-09T06:26:45.047540+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:32.947543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":166,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057476+0000","last_change":"2023-10-09T06:18:59.818258+0000","last_active":"2023-10-09T06:26:45.057476+0000","last_peered":"2023-10-09T06:26:45.057476+0000","last_clean":"2023-10-09T06:26:45.057476+0000","last_became_active":"2023-10-09T06:18:59.811399+0000","last_became_peered":"2023-10-09T06:18:59.811399+0000","last_unstale":"2023-10-09T06:26:45.057476+0000","last_undegraded":"2023-10-09T06:26:45.057476+0000","last_fullsized":"2023-10-09T06:26:45.057476+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":120,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057122+0000","last_change":"2023-10-09T06:13:43.441656+0000","last_active":"2023-10-09T06:26:45.057122+0000","last_peered":"2023-10-09T06:26:45.057122+0000","last_clean":"2023-10-09T06:26:45.057122+0000","last_became_active":"2023-10-09T06:13:43.441189+0000","last_became_peered":"2023-10-09T06:13:43.441189+0000","last_unstale":"2023-10-09T06:26:45.057122+0000","last_undegraded":"2023-10-09T06:26:45.057122+0000","last_fullsized":"2023-10-09T06:26:45.057122+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":54,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.065984+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:26:45.065984+0000","last_peered":"2023-10-09T06:26:45.065984+0000","last_clean":"2023-10-09T06:26:45.065984+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:26:45.065984+0000","last_undegraded":"2023-10-09T06:26:45.065984+0000","last_fullsized":"2023-10-09T06:26:45.065984+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":147,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056024+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:26:45.056024+0000","last_peered":"2023-10-09T06:26:45.056024+0000","last_clean":"2023-10-09T06:26:45.056024+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:26:45.056024+0000","last_undegraded":"2023-10-09T06:26:45.056024+0000","last_fullsized":"2023-10-09T06:26:45.056024+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:12.048095+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:27:12.048095+0000","last_peered":"2023-10-09T06:27:12.048095+0000","last_clean":"2023-10-09T06:27:12.048095+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:27:12.048095+0000","last_undegraded":"2023-10-09T06:27:12.048095+0000","last_fullsized":"2023-10-09T06:27:12.048095+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.062984+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:26:45.062984+0000","last_peered":"2023-10-09T06:26:45.062984+0000","last_clean":"2023-10-09T06:26:45.062984+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:26:45.062984+0000","last_undegraded":"2023-10-09T06:26:45.062984+0000","last_fullsized":"2023-10-09T06:26:45.062984+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":48,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.065007+0000","last_change":"2023-10-09T06:18:59.812207+0000","last_active":"2023-10-09T06:26:45.065007+0000","last_peered":"2023-10-09T06:26:45.065007+0000","last_clean":"2023-10-09T06:26:45.065007+0000","last_became_active":"2023-10-09T06:18:59.810860+0000","last_became_peered":"2023-10-09T06:18:59.810860+0000","last_unstale":"2023-10-09T06:26:45.065007+0000","last_undegraded":"2023-10-09T06:26:45.065007+0000","last_fullsized":"2023-10-09T06:26:45.065007+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":126,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.178746+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:26:45.178746+0000","last_peered":"2023-10-09T06:26:45.178746+0000","last_clean":"2023-10-09T06:26:45.178746+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:26:45.178746+0000","last_undegraded":"2023-10-09T06:26:45.178746+0000","last_fullsized":"2023-10-09T06:26:45.178746+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":130,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.053708+0000","last_change":"2023-10-09T06:13:14.651583+0000","last_active":"2023-10-09T06:26:45.053708+0000","last_peered":"2023-10-09T06:26:45.053708+0000","last_clean":"2023-10-09T06:26:45.053708+0000","last_became_active":"2023-10-09T06:13:14.651169+0000","last_became_peered":"2023-10-09T06:13:14.651169+0000","last_unstale":"2023-10-09T06:26:45.053708+0000","last_undegraded":"2023-10-09T06:26:45.053708+0000","last_fullsized":"2023-10-09T06:26:45.053708+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":160,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:46.571789+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:26:46.571789+0000","last_peered":"2023-10-09T06:26:46.571789+0000","last_clean":"2023-10-09T06:26:46.571789+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:26:46.571789+0000","last_undegraded":"2023-10-09T06:26:46.571789+0000","last_fullsized":"2023-10-09T06:26:46.571789+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"156'13870","reported_seq":13892,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.484980+0000","last_change":"2023-10-09T06:23:02.818534+0000","last_active":"2023-10-09T06:27:29.484980+0000","last_peered":"2023-10-09T06:27:29.484980+0000","last_clean":"2023-10-09T06:27:29.484980+0000","last_became_active":"2023-10-09T06:23:02.816918+0000","last_became_peered":"2023-10-09T06:23:02.816918+0000","last_unstale":"2023-10-09T06:27:29.484980+0000","last_undegraded":"2023-10-09T06:27:29.484980+0000","last_fullsized":"2023-10-09T06:27:29.484980+0000","mapping_epoch":142,"log_start":"147'3800","ondisk_log_start":"147'3800","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13870,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":52,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063759+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:26:45.063759+0000","last_peered":"2023-10-09T06:26:45.063759+0000","last_clean":"2023-10-09T06:26:45.063759+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:26:45.063759+0000","last_undegraded":"2023-10-09T06:26:45.063759+0000","last_fullsized":"2023-10-09T06:26:45.063759+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057785+0000","last_change":"2023-10-09T06:18:59.804330+0000","last_active":"2023-10-09T06:26:45.057785+0000","last_peered":"2023-10-09T06:26:45.057785+0000","last_clean":"2023-10-09T06:26:45.057785+0000","last_became_active":"2023-10-09T06:18:59.803937+0000","last_became_peered":"2023-10-09T06:18:59.803937+0000","last_unstale":"2023-10-09T06:26:45.057785+0000","last_undegraded":"2023-10-09T06:26:45.057785+0000","last_fullsized":"2023-10-09T06:26:45.057785+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"156'13930","reported_seq":13954,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.100799+0000","last_change":"2023-10-09T06:23:02.810028+0000","last_active":"2023-10-09T06:27:30.100799+0000","last_peered":"2023-10-09T06:27:30.100799+0000","last_clean":"2023-10-09T06:27:30.100799+0000","last_became_active":"2023-10-09T06:23:02.809179+0000","last_became_peered":"2023-10-09T06:23:02.809179+0000","last_unstale":"2023-10-09T06:27:30.100799+0000","last_undegraded":"2023-10-09T06:27:30.100799+0000","last_fullsized":"2023-10-09T06:27:30.100799+0000","mapping_epoch":142,"log_start":"149'6600","ondisk_log_start":"149'6600","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:49.851184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13930,"num_write_kb":445760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":52,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063727+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:26:45.063727+0000","last_peered":"2023-10-09T06:26:45.063727+0000","last_clean":"2023-10-09T06:26:45.063727+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:26:45.063727+0000","last_undegraded":"2023-10-09T06:26:45.063727+0000","last_fullsized":"2023-10-09T06:26:45.063727+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056756+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:26:45.056756+0000","last_peered":"2023-10-09T06:26:45.056756+0000","last_clean":"2023-10-09T06:26:45.056756+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:26:45.056756+0000","last_undegraded":"2023-10-09T06:26:45.056756+0000","last_fullsized":"2023-10-09T06:26:45.056756+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"156'13797","reported_seq":13819,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.613086+0000","last_change":"2023-10-09T06:23:02.818119+0000","last_active":"2023-10-09T06:27:29.613086+0000","last_peered":"2023-10-09T06:27:29.613086+0000","last_clean":"2023-10-09T06:27:29.613086+0000","last_became_active":"2023-10-09T06:23:02.817802+0000","last_became_peered":"2023-10-09T06:23:02.817802+0000","last_unstale":"2023-10-09T06:27:29.613086+0000","last_undegraded":"2023-10-09T06:27:29.613086+0000","last_fullsized":"2023-10-09T06:27:29.613086+0000","mapping_epoch":142,"log_start":"148'5400","ondisk_log_start":"148'5400","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1507328,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13797,"num_write_kb":442240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":52,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063616+0000","last_change":"2023-10-09T06:18:59.904584+0000","last_active":"2023-10-09T06:26:45.063616+0000","last_peered":"2023-10-09T06:26:45.063616+0000","last_clean":"2023-10-09T06:26:45.063616+0000","last_became_active":"2023-10-09T06:18:59.807104+0000","last_became_peered":"2023-10-09T06:18:59.807104+0000","last_unstale":"2023-10-09T06:26:45.063616+0000","last_undegraded":"2023-10-09T06:26:45.063616+0000","last_fullsized":"2023-10-09T06:26:45.063616+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.054023+0000","last_change":"2023-10-09T06:18:59.807692+0000","last_active":"2023-10-09T06:26:45.054023+0000","last_peered":"2023-10-09T06:26:45.054023+0000","last_clean":"2023-10-09T06:26:45.054023+0000","last_became_active":"2023-10-09T06:18:59.807336+0000","last_became_peered":"2023-10-09T06:18:59.807336+0000","last_unstale":"2023-10-09T06:26:45.054023+0000","last_undegraded":"2023-10-09T06:26:45.054023+0000","last_fullsized":"2023-10-09T06:26:45.054023+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"156'13860","reported_seq":13885,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.122871+0000","last_change":"2023-10-09T06:23:02.807021+0000","last_active":"2023-10-09T06:27:30.122871+0000","last_peered":"2023-10-09T06:27:30.122871+0000","last_clean":"2023-10-09T06:27:30.122871+0000","last_became_active":"2023-10-09T06:23:02.806707+0000","last_became_peered":"2023-10-09T06:23:02.806707+0000","last_unstale":"2023-10-09T06:27:30.122871+0000","last_undegraded":"2023-10-09T06:27:30.122871+0000","last_fullsized":"2023-10-09T06:27:30.122871+0000","mapping_epoch":142,"log_start":"147'3800","ondisk_log_start":"147'3800","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10060,"log_dups_size":0,"ondisk_log_size":10060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:19.835325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13860,"num_write_kb":443520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.058139+0000","last_change":"2023-10-09T06:14:42.691189+0000","last_active":"2023-10-09T06:26:45.058139+0000","last_peered":"2023-10-09T06:26:45.058139+0000","last_clean":"2023-10-09T06:26:45.058139+0000","last_became_active":"2023-10-09T06:14:42.690811+0000","last_became_peered":"2023-10-09T06:14:42.690811+0000","last_unstale":"2023-10-09T06:26:45.058139+0000","last_undegraded":"2023-10-09T06:26:45.058139+0000","last_fullsized":"2023-10-09T06:26:45.058139+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":109,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.178506+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:26:45.178506+0000","last_peered":"2023-10-09T06:26:45.178506+0000","last_clean":"2023-10-09T06:26:45.178506+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:26:45.178506+0000","last_undegraded":"2023-10-09T06:26:45.178506+0000","last_fullsized":"2023-10-09T06:26:45.178506+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"156'13773","reported_seq":13795,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.588884+0000","last_change":"2023-10-09T06:23:02.820523+0000","last_active":"2023-10-09T06:27:29.588884+0000","last_peered":"2023-10-09T06:27:29.588884+0000","last_clean":"2023-10-09T06:27:29.588884+0000","last_became_active":"2023-10-09T06:23:02.820107+0000","last_became_peered":"2023-10-09T06:23:02.820107+0000","last_unstale":"2023-10-09T06:27:29.588884+0000","last_undegraded":"2023-10-09T06:27:29.588884+0000","last_fullsized":"2023-10-09T06:27:29.588884+0000","mapping_epoch":142,"log_start":"148'5800","ondisk_log_start":"148'5800","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13773,"num_write_kb":441408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":128,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.059020+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:26:45.059020+0000","last_peered":"2023-10-09T06:26:45.059020+0000","last_clean":"2023-10-09T06:26:45.059020+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:26:45.059020+0000","last_undegraded":"2023-10-09T06:26:45.059020+0000","last_fullsized":"2023-10-09T06:26:45.059020+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":120,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057392+0000","last_change":"2023-10-09T06:14:42.689434+0000","last_active":"2023-10-09T06:26:45.057392+0000","last_peered":"2023-10-09T06:26:45.057392+0000","last_clean":"2023-10-09T06:26:45.057392+0000","last_became_active":"2023-10-09T06:14:42.689066+0000","last_became_peered":"2023-10-09T06:14:42.689066+0000","last_unstale":"2023-10-09T06:26:45.057392+0000","last_undegraded":"2023-10-09T06:26:45.057392+0000","last_fullsized":"2023-10-09T06:26:45.057392+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"156'13648","reported_seq":13672,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.113919+0000","last_change":"2023-10-09T06:23:02.812031+0000","last_active":"2023-10-09T06:27:30.113919+0000","last_peered":"2023-10-09T06:27:30.113919+0000","last_clean":"2023-10-09T06:27:30.113919+0000","last_became_active":"2023-10-09T06:23:02.811742+0000","last_became_peered":"2023-10-09T06:23:02.811742+0000","last_unstale":"2023-10-09T06:27:30.113919+0000","last_undegraded":"2023-10-09T06:27:30.113919+0000","last_fullsized":"2023-10-09T06:27:30.113919+0000","mapping_epoch":142,"log_start":"147'3600","ondisk_log_start":"147'3600","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:34.791030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13648,"num_write_kb":436736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":68,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060836+0000","last_change":"2023-10-09T06:25:30.035782+0000","last_active":"2023-10-09T06:26:45.060836+0000","last_peered":"2023-10-09T06:26:45.060836+0000","last_clean":"2023-10-09T06:26:45.060836+0000","last_became_active":"2023-10-09T06:25:30.035376+0000","last_became_peered":"2023-10-09T06:25:30.035376+0000","last_unstale":"2023-10-09T06:26:45.060836+0000","last_undegraded":"2023-10-09T06:26:45.060836+0000","last_fullsized":"2023-10-09T06:26:45.060836+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060515+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:26:45.060515+0000","last_peered":"2023-10-09T06:26:45.060515+0000","last_clean":"2023-10-09T06:26:45.060515+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:26:45.060515+0000","last_undegraded":"2023-10-09T06:26:45.060515+0000","last_fullsized":"2023-10-09T06:26:45.060515+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"156'13984","reported_seq":14008,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.146628+0000","last_change":"2023-10-09T06:23:02.813423+0000","last_active":"2023-10-09T06:27:30.146628+0000","last_peered":"2023-10-09T06:27:30.146628+0000","last_clean":"2023-10-09T06:27:30.146628+0000","last_became_active":"2023-10-09T06:23:02.813075+0000","last_became_peered":"2023-10-09T06:23:02.813075+0000","last_unstale":"2023-10-09T06:27:30.146628+0000","last_undegraded":"2023-10-09T06:27:30.146628+0000","last_fullsized":"2023-10-09T06:27:30.146628+0000","mapping_epoch":142,"log_start":"149'6600","ondisk_log_start":"149'6600","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:07.790690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13984,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056564+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:26:45.056564+0000","last_peered":"2023-10-09T06:26:45.056564+0000","last_clean":"2023-10-09T06:26:45.056564+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:26:45.056564+0000","last_undegraded":"2023-10-09T06:26:45.056564+0000","last_fullsized":"2023-10-09T06:26:45.056564+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":120,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060859+0000","last_change":"2023-10-09T06:13:43.440626+0000","last_active":"2023-10-09T06:26:45.060859+0000","last_peered":"2023-10-09T06:26:45.060859+0000","last_clean":"2023-10-09T06:26:45.060859+0000","last_became_active":"2023-10-09T06:13:43.440317+0000","last_became_peered":"2023-10-09T06:13:43.440317+0000","last_unstale":"2023-10-09T06:26:45.060859+0000","last_undegraded":"2023-10-09T06:26:45.060859+0000","last_fullsized":"2023-10-09T06:26:45.060859+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"156'13782","reported_seq":13805,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.541943+0000","last_change":"2023-10-09T06:23:02.819776+0000","last_active":"2023-10-09T06:27:29.541943+0000","last_peered":"2023-10-09T06:27:29.541943+0000","last_clean":"2023-10-09T06:27:29.541943+0000","last_became_active":"2023-10-09T06:23:02.819408+0000","last_became_peered":"2023-10-09T06:23:02.819408+0000","last_unstale":"2023-10-09T06:27:29.541943+0000","last_undegraded":"2023-10-09T06:27:29.541943+0000","last_fullsized":"2023-10-09T06:27:29.541943+0000","mapping_epoch":142,"log_start":"147'3700","ondisk_log_start":"147'3700","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638424,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13782,"num_write_kb":441793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":148,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.053864+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:26:45.053864+0000","last_peered":"2023-10-09T06:26:45.053864+0000","last_clean":"2023-10-09T06:26:45.053864+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:26:45.053864+0000","last_undegraded":"2023-10-09T06:26:45.053864+0000","last_fullsized":"2023-10-09T06:26:45.053864+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:27:11.950662+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:27:11.950662+0000","last_peered":"2023-10-09T06:27:11.950662+0000","last_clean":"2023-10-09T06:27:11.950662+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:27:11.950662+0000","last_undegraded":"2023-10-09T06:27:11.950662+0000","last_fullsized":"2023-10-09T06:27:11.950662+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"156'13920","reported_seq":13945,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.112442+0000","last_change":"2023-10-09T06:23:02.808018+0000","last_active":"2023-10-09T06:27:30.112442+0000","last_peered":"2023-10-09T06:27:30.112442+0000","last_clean":"2023-10-09T06:27:30.112442+0000","last_became_active":"2023-10-09T06:23:02.807707+0000","last_became_peered":"2023-10-09T06:23:02.807707+0000","last_unstale":"2023-10-09T06:27:30.112442+0000","last_undegraded":"2023-10-09T06:27:30.112442+0000","last_fullsized":"2023-10-09T06:27:30.112442+0000","mapping_epoch":142,"log_start":"147'3900","ondisk_log_start":"147'3900","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:53.598902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13920,"num_write_kb":445440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1006,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.049424+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:26:45.049424+0000","last_peered":"2023-10-09T06:26:45.049424+0000","last_clean":"2023-10-09T06:26:45.049424+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:26:45.049424+0000","last_undegraded":"2023-10-09T06:26:45.049424+0000","last_fullsized":"2023-10-09T06:26:45.049424+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063086+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:26:45.063086+0000","last_peered":"2023-10-09T06:26:45.063086+0000","last_clean":"2023-10-09T06:26:45.063086+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:26:45.063086+0000","last_undegraded":"2023-10-09T06:26:45.063086+0000","last_fullsized":"2023-10-09T06:26:45.063086+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.047468+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:26:45.047468+0000","last_peered":"2023-10-09T06:26:45.047468+0000","last_clean":"2023-10-09T06:26:45.047468+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:26:45.047468+0000","last_undegraded":"2023-10-09T06:26:45.047468+0000","last_fullsized":"2023-10-09T06:26:45.047468+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"156'13974","reported_seq":13999,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.140687+0000","last_change":"2023-10-09T06:23:02.809922+0000","last_active":"2023-10-09T06:27:30.140687+0000","last_peered":"2023-10-09T06:27:30.140687+0000","last_clean":"2023-10-09T06:27:30.140687+0000","last_became_active":"2023-10-09T06:23:02.809657+0000","last_became_peered":"2023-10-09T06:23:02.809657+0000","last_unstale":"2023-10-09T06:27:30.140687+0000","last_undegraded":"2023-10-09T06:27:30.140687+0000","last_fullsized":"2023-10-09T06:27:30.140687+0000","mapping_epoch":142,"log_start":"149'6600","ondisk_log_start":"149'6600","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:22.193166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13974,"num_write_kb":447168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":737,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.061839+0000","last_change":"2023-10-09T06:13:43.439144+0000","last_active":"2023-10-09T06:26:45.061839+0000","last_peered":"2023-10-09T06:26:45.061839+0000","last_clean":"2023-10-09T06:26:45.061839+0000","last_became_active":"2023-10-09T06:13:43.438714+0000","last_became_peered":"2023-10-09T06:13:43.438714+0000","last_unstale":"2023-10-09T06:26:45.061839+0000","last_undegraded":"2023-10-09T06:26:45.061839+0000","last_fullsized":"2023-10-09T06:26:45.061839+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057052+0000","last_change":"2023-10-09T06:13:14.653858+0000","last_active":"2023-10-09T06:26:45.057052+0000","last_peered":"2023-10-09T06:26:45.057052+0000","last_clean":"2023-10-09T06:26:45.057052+0000","last_became_active":"2023-10-09T06:13:14.653577+0000","last_became_peered":"2023-10-09T06:13:14.653577+0000","last_unstale":"2023-10-09T06:26:45.057052+0000","last_undegraded":"2023-10-09T06:26:45.057052+0000","last_fullsized":"2023-10-09T06:26:45.057052+0000","mapping_epoch":88,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":89,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.049269+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:26:45.049269+0000","last_peered":"2023-10-09T06:26:45.049269+0000","last_clean":"2023-10-09T06:26:45.049269+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:26:45.049269+0000","last_undegraded":"2023-10-09T06:26:45.049269+0000","last_fullsized":"2023-10-09T06:26:45.049269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"156'13656","reported_seq":13678,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.605572+0000","last_change":"2023-10-09T06:23:02.818151+0000","last_active":"2023-10-09T06:27:29.605572+0000","last_peered":"2023-10-09T06:27:29.605572+0000","last_clean":"2023-10-09T06:27:29.605572+0000","last_became_active":"2023-10-09T06:23:02.816332+0000","last_became_peered":"2023-10-09T06:23:02.816332+0000","last_unstale":"2023-10-09T06:27:29.605572+0000","last_undegraded":"2023-10-09T06:27:29.605572+0000","last_fullsized":"2023-10-09T06:27:29.605572+0000","mapping_epoch":142,"log_start":"148'5300","ondisk_log_start":"148'5300","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":8356,"log_dups_size":0,"ondisk_log_size":8356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13656,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":517,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.061426+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:26:45.061426+0000","last_peered":"2023-10-09T06:26:45.061426+0000","last_clean":"2023-10-09T06:26:45.061426+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:26:45.061426+0000","last_undegraded":"2023-10-09T06:26:45.061426+0000","last_fullsized":"2023-10-09T06:26:45.061426+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.046855+0000","last_change":"2023-10-09T06:14:12.888217+0000","last_active":"2023-10-09T06:26:45.046855+0000","last_peered":"2023-10-09T06:26:45.046855+0000","last_clean":"2023-10-09T06:26:45.046855+0000","last_became_active":"2023-10-09T06:14:12.887924+0000","last_became_peered":"2023-10-09T06:14:12.887924+0000","last_unstale":"2023-10-09T06:26:45.046855+0000","last_undegraded":"2023-10-09T06:26:45.046855+0000","last_fullsized":"2023-10-09T06:26:45.046855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:02:44.497954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.048613+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:26:45.048613+0000","last_peered":"2023-10-09T06:26:45.048613+0000","last_clean":"2023-10-09T06:26:45.048613+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:26:45.048613+0000","last_undegraded":"2023-10-09T06:26:45.048613+0000","last_fullsized":"2023-10-09T06:26:45.048613+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"156'14057","reported_seq":14078,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.556926+0000","last_change":"2023-10-09T06:23:02.821017+0000","last_active":"2023-10-09T06:27:29.556926+0000","last_peered":"2023-10-09T06:27:29.556926+0000","last_clean":"2023-10-09T06:27:29.556926+0000","last_became_active":"2023-10-09T06:23:02.820708+0000","last_became_peered":"2023-10-09T06:23:02.820708+0000","last_unstale":"2023-10-09T06:27:29.556926+0000","last_undegraded":"2023-10-09T06:27:29.556926+0000","last_fullsized":"2023-10-09T06:27:29.556926+0000","mapping_epoch":142,"log_start":"147'4000","ondisk_log_start":"147'4000","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14057,"num_write_kb":450368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":504,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057929+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:26:45.057929+0000","last_peered":"2023-10-09T06:26:45.057929+0000","last_clean":"2023-10-09T06:26:45.057929+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:26:45.057929+0000","last_undegraded":"2023-10-09T06:26:45.057929+0000","last_fullsized":"2023-10-09T06:26:45.057929+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":352,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063626+0000","last_change":"2023-10-09T06:19:02.846110+0000","last_active":"2023-10-09T06:26:45.063626+0000","last_peered":"2023-10-09T06:26:45.063626+0000","last_clean":"2023-10-09T06:26:45.063626+0000","last_became_active":"2023-10-09T06:19:02.845481+0000","last_became_peered":"2023-10-09T06:19:02.845481+0000","last_unstale":"2023-10-09T06:26:45.063626+0000","last_undegraded":"2023-10-09T06:26:45.063626+0000","last_fullsized":"2023-10-09T06:26:45.063626+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:27:11.628812+0000","last_change":"2023-10-09T06:14:42.688699+0000","last_active":"2023-10-09T06:27:11.628812+0000","last_peered":"2023-10-09T06:27:11.628812+0000","last_clean":"2023-10-09T06:27:11.628812+0000","last_became_active":"2023-10-09T06:14:42.687991+0000","last_became_peered":"2023-10-09T06:14:42.687991+0000","last_unstale":"2023-10-09T06:27:11.628812+0000","last_undegraded":"2023-10-09T06:27:11.628812+0000","last_fullsized":"2023-10-09T06:27:11.628812+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:52.731567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060686+0000","last_change":"2023-10-09T06:25:30.049123+0000","last_active":"2023-10-09T06:26:45.060686+0000","last_peered":"2023-10-09T06:26:45.060686+0000","last_clean":"2023-10-09T06:26:45.060686+0000","last_became_active":"2023-10-09T06:25:30.048779+0000","last_became_peered":"2023-10-09T06:25:30.048779+0000","last_unstale":"2023-10-09T06:26:45.060686+0000","last_undegraded":"2023-10-09T06:26:45.060686+0000","last_fullsized":"2023-10-09T06:26:45.060686+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"156'14166","reported_seq":14191,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.137154+0000","last_change":"2023-10-09T06:23:02.811831+0000","last_active":"2023-10-09T06:27:30.137154+0000","last_peered":"2023-10-09T06:27:30.137154+0000","last_clean":"2023-10-09T06:27:30.137154+0000","last_became_active":"2023-10-09T06:23:02.811445+0000","last_became_peered":"2023-10-09T06:23:02.811445+0000","last_unstale":"2023-10-09T06:27:30.137154+0000","last_undegraded":"2023-10-09T06:27:30.137154+0000","last_fullsized":"2023-10-09T06:27:30.137154+0000","mapping_epoch":142,"log_start":"148'6000","ondisk_log_start":"148'6000","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:28.021647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14166,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":77,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.065754+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:26:45.065754+0000","last_peered":"2023-10-09T06:26:45.065754+0000","last_clean":"2023-10-09T06:26:45.065754+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:26:45.065754+0000","last_undegraded":"2023-10-09T06:26:45.065754+0000","last_fullsized":"2023-10-09T06:26:45.065754+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":109,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.178507+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:26:45.178507+0000","last_peered":"2023-10-09T06:26:45.178507+0000","last_clean":"2023-10-09T06:26:45.178507+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:26:45.178507+0000","last_undegraded":"2023-10-09T06:26:45.178507+0000","last_fullsized":"2023-10-09T06:26:45.178507+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.179151+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:26:45.179151+0000","last_peered":"2023-10-09T06:26:45.179151+0000","last_clean":"2023-10-09T06:26:45.179151+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:26:45.179151+0000","last_undegraded":"2023-10-09T06:26:45.179151+0000","last_fullsized":"2023-10-09T06:26:45.179151+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"156'13848","reported_seq":13873,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:30.140211+0000","last_change":"2023-10-09T06:23:02.810207+0000","last_active":"2023-10-09T06:27:30.140211+0000","last_peered":"2023-10-09T06:27:30.140211+0000","last_clean":"2023-10-09T06:27:30.140211+0000","last_became_active":"2023-10-09T06:23:02.809844+0000","last_became_peered":"2023-10-09T06:23:02.809844+0000","last_unstale":"2023-10-09T06:27:30.140211+0000","last_undegraded":"2023-10-09T06:27:30.140211+0000","last_fullsized":"2023-10-09T06:27:30.140211+0000","mapping_epoch":142,"log_start":"148'5700","ondisk_log_start":"148'5700","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:38.667224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13848,"num_write_kb":443136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":66,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063507+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:26:45.063507+0000","last_peered":"2023-10-09T06:26:45.063507+0000","last_clean":"2023-10-09T06:26:45.063507+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:26:45.063507+0000","last_undegraded":"2023-10-09T06:26:45.063507+0000","last_fullsized":"2023-10-09T06:26:45.063507+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.049349+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:26:45.049349+0000","last_peered":"2023-10-09T06:26:45.049349+0000","last_clean":"2023-10-09T06:26:45.049349+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:26:45.049349+0000","last_undegraded":"2023-10-09T06:26:45.049349+0000","last_fullsized":"2023-10-09T06:26:45.049349+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":121,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.178830+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:26:45.178830+0000","last_peered":"2023-10-09T06:26:45.178830+0000","last_clean":"2023-10-09T06:26:45.178830+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:26:45.178830+0000","last_undegraded":"2023-10-09T06:26:45.178830+0000","last_fullsized":"2023-10-09T06:26:45.178830+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"156'13899","reported_seq":13921,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.524666+0000","last_change":"2023-10-09T06:23:02.821182+0000","last_active":"2023-10-09T06:27:29.524666+0000","last_peered":"2023-10-09T06:27:29.524666+0000","last_clean":"2023-10-09T06:27:29.524666+0000","last_became_active":"2023-10-09T06:23:02.820839+0000","last_became_peered":"2023-10-09T06:23:02.820839+0000","last_unstale":"2023-10-09T06:27:29.524666+0000","last_undegraded":"2023-10-09T06:27:29.524666+0000","last_fullsized":"2023-10-09T06:27:29.524666+0000","mapping_epoch":142,"log_start":"148'6000","ondisk_log_start":"148'6000","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1769472,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13899,"num_write_kb":445632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":154,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.059974+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:26:45.059974+0000","last_peered":"2023-10-09T06:26:45.059974+0000","last_clean":"2023-10-09T06:26:45.059974+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:26:45.059974+0000","last_undegraded":"2023-10-09T06:26:45.059974+0000","last_fullsized":"2023-10-09T06:26:45.059974+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":462,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.047895+0000","last_change":"2023-10-09T06:13:43.441044+0000","last_active":"2023-10-09T06:26:45.047895+0000","last_peered":"2023-10-09T06:26:45.047895+0000","last_clean":"2023-10-09T06:26:45.047895+0000","last_became_active":"2023-10-09T06:13:43.440373+0000","last_became_peered":"2023-10-09T06:13:43.440373+0000","last_unstale":"2023-10-09T06:26:45.047895+0000","last_undegraded":"2023-10-09T06:26:45.047895+0000","last_fullsized":"2023-10-09T06:26:45.047895+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:54.486210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":153,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.053882+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:26:45.053882+0000","last_peered":"2023-10-09T06:26:45.053882+0000","last_clean":"2023-10-09T06:26:45.053882+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:26:45.053882+0000","last_undegraded":"2023-10-09T06:26:45.053882+0000","last_fullsized":"2023-10-09T06:26:45.053882+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"156'14035","reported_seq":14056,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-10-09T06:27:29.604746+0000","last_change":"2023-10-09T06:23:02.819624+0000","last_active":"2023-10-09T06:27:29.604746+0000","last_peered":"2023-10-09T06:27:29.604746+0000","last_clean":"2023-10-09T06:27:29.604746+0000","last_became_active":"2023-10-09T06:23:02.818630+0000","last_became_peered":"2023-10-09T06:23:02.818630+0000","last_unstale":"2023-10-09T06:27:29.604746+0000","last_undegraded":"2023-10-09T06:27:29.604746+0000","last_fullsized":"2023-10-09T06:27:29.604746+0000","mapping_epoch":142,"log_start":"148'5700","ondisk_log_start":"148'5700","created":142,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:23:01.781797+0000","last_clean_scrub_stamp":"2023-10-09T06:23:01.781797+0000","objects_scrubbed":0,"log_size":8335,"log_dups_size":0,"ondisk_log_size":8335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1769472,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14035,"num_write_kb":449984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":852,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.054037+0000","last_change":"2023-10-09T06:22:46.890972+0000","last_active":"2023-10-09T06:26:45.054037+0000","last_peered":"2023-10-09T06:26:45.054037+0000","last_clean":"2023-10-09T06:26:45.054037+0000","last_became_active":"2023-10-09T06:22:46.889916+0000","last_became_peered":"2023-10-09T06:22:46.889916+0000","last_unstale":"2023-10-09T06:26:45.054037+0000","last_undegraded":"2023-10-09T06:26:45.054037+0000","last_fullsized":"2023-10-09T06:26:45.054037+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":113,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.179064+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:26:45.179064+0000","last_peered":"2023-10-09T06:26:45.179064+0000","last_clean":"2023-10-09T06:26:45.179064+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:26:45.179064+0000","last_undegraded":"2023-10-09T06:26:45.179064+0000","last_fullsized":"2023-10-09T06:26:45.179064+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056635+0000","last_change":"2023-10-09T06:01:50.670140+0000","last_active":"2023-10-09T06:26:45.056635+0000","last_peered":"2023-10-09T06:26:45.056635+0000","last_clean":"2023-10-09T06:26:45.056635+0000","last_became_active":"2023-10-09T06:01:50.669700+0000","last_became_peered":"2023-10-09T06:01:50.669700+0000","last_unstale":"2023-10-09T06:26:45.056635+0000","last_undegraded":"2023-10-09T06:26:45.056635+0000","last_fullsized":"2023-10-09T06:26:45.056635+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":115,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.178757+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:26:45.178757+0000","last_peered":"2023-10-09T06:26:45.178757+0000","last_clean":"2023-10-09T06:26:45.178757+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:26:45.178757+0000","last_undegraded":"2023-10-09T06:26:45.178757+0000","last_fullsized":"2023-10-09T06:26:45.178757+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.061965+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:26:45.061965+0000","last_peered":"2023-10-09T06:26:45.061965+0000","last_clean":"2023-10-09T06:26:45.061965+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:26:45.061965+0000","last_undegraded":"2023-10-09T06:26:45.061965+0000","last_fullsized":"2023-10-09T06:26:45.061965+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":123,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060943+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:26:45.060943+0000","last_peered":"2023-10-09T06:26:45.060943+0000","last_clean":"2023-10-09T06:26:45.060943+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:26:45.060943+0000","last_undegraded":"2023-10-09T06:26:45.060943+0000","last_fullsized":"2023-10-09T06:26:45.060943+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.057249+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:26:45.057249+0000","last_peered":"2023-10-09T06:26:45.057249+0000","last_clean":"2023-10-09T06:26:45.057249+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:26:45.057249+0000","last_undegraded":"2023-10-09T06:26:45.057249+0000","last_fullsized":"2023-10-09T06:26:45.057249+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.063185+0000","last_change":"2023-10-09T06:01:50.650447+0000","last_active":"2023-10-09T06:26:45.063185+0000","last_peered":"2023-10-09T06:26:45.063185+0000","last_clean":"2023-10-09T06:26:45.063185+0000","last_became_active":"2023-10-09T06:01:50.650116+0000","last_became_peered":"2023-10-09T06:01:50.650116+0000","last_unstale":"2023-10-09T06:26:45.063185+0000","last_undegraded":"2023-10-09T06:26:45.063185+0000","last_fullsized":"2023-10-09T06:26:45.063185+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.060937+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:26:45.060937+0000","last_peered":"2023-10-09T06:26:45.060937+0000","last_clean":"2023-10-09T06:26:45.060937+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:26:45.060937+0000","last_undegraded":"2023-10-09T06:26:45.060937+0000","last_fullsized":"2023-10-09T06:26:45.060937+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056086+0000","last_change":"2023-10-09T06:14:42.688599+0000","last_active":"2023-10-09T06:26:45.056086+0000","last_peered":"2023-10-09T06:26:45.056086+0000","last_clean":"2023-10-09T06:26:45.056086+0000","last_became_active":"2023-10-09T06:14:42.687734+0000","last_became_peered":"2023-10-09T06:14:42.687734+0000","last_unstale":"2023-10-09T06:26:45.056086+0000","last_undegraded":"2023-10-09T06:26:45.056086+0000","last_fullsized":"2023-10-09T06:26:45.056086+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:33.838010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":148,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:46.568925+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:26:46.568925+0000","last_peered":"2023-10-09T06:26:46.568925+0000","last_clean":"2023-10-09T06:26:46.568925+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:26:46.568925+0000","last_undegraded":"2023-10-09T06:26:46.568925+0000","last_fullsized":"2023-10-09T06:26:46.568925+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":113,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.179115+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:26:45.179115+0000","last_peered":"2023-10-09T06:26:45.179115+0000","last_clean":"2023-10-09T06:26:45.179115+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:26:45.179115+0000","last_undegraded":"2023-10-09T06:26:45.179115+0000","last_fullsized":"2023-10-09T06:26:45.179115+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-09T06:26:45.056459+0000","last_change":"2023-10-09T06:13:43.440279+0000","last_active":"2023-10-09T06:26:45.056459+0000","last_peered":"2023-10-09T06:26:45.056459+0000","last_clean":"2023-10-09T06:26:45.056459+0000","last_became_active":"2023-10-09T06:13:43.439921+0000","last_became_peered":"2023-10-09T06:13:43.439921+0000","last_unstale":"2023-10-09T06:26:45.056459+0000","last_undegraded":"2023-10-09T06:26:45.056459+0000","last_fullsized":"2023-10-09T06:26:45.056459+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":11665432,"num_objects":179,"num_object_clones":0,"num_object_copies":537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":179,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":222199,"num_write_kb":7116033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":29167616,"data_stored":29163544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139699,"ondisk_log_size":139699,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430554,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":326740,"kb_used_data":17228,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93418412,"statfs":{"total":95995035648,"available":95660453888,"internally_reserved":0,"allocated":17641472,"data_stored":17111309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:35 2023","interfaces":[{"interface":"back","average":{"1min":4.843,"5min":2.4790000000000001,"15min":1.883},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":73.491,"5min":73.491,"15min":107.36799999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":4.899,"5min":2.4319999999999999,"15min":1.861},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":73.846999999999994,"5min":73.846999999999994,"15min":105.89},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":2.1200000000000001,"15min":1.9359999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.8610000000000007,"5min":63.252000000000002,"15min":104.825},"last":0.625},{"interface":"front","average":{"1min":2.234,"5min":2.0409999999999999,"15min":1.7949999999999999},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2530000000000001,"5min":60.746000000000002,"15min":104.637},"last":0.59799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:27:28 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.778,"15min":2.2719999999999998},"min":{"1min":0.34300000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.5399999999999991,"5min":12.789999999999999,"15min":104.992},"last":1.8999999999999999},{"interface":"front","average":{"1min":2.2879999999999998,"5min":1.5860000000000001,"15min":2.21},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3339999999999996,"5min":6.2770000000000001,"15min":104.691},"last":1.9370000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":4.0129999999999999,"15min":2.1190000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.5350000000000001,"5min":137.11600000000001,"15min":137.11600000000001},"last":2.3239999999999998},{"interface":"front","average":{"1min":1.675,"5min":3.7549999999999999,"15min":2.0190000000000001},"min":{"1min":0.34599999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3849999999999998,"5min":137.14400000000001,"15min":137.14400000000001},"last":2.3450000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.24,"15min":1.1919999999999999},"min":{"1min":0.35099999999999998,"5min":0.30499999999999999,"15min":0.29399999999999998},"max":{"1min":3.044,"5min":11.029999999999999,"15min":11.029999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.7470000000000001,"15min":1.429},"min":{"1min":0.371,"5min":0.307,"15min":0.307},"max":{"1min":8.5850000000000009,"5min":8.9800000000000004,"15min":8.9800000000000004},"last":0.60599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":1.415,"5min":1.2969999999999999,"15min":1.3720000000000001},"min":{"1min":0.47599999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.6129999999999995,"5min":8.6129999999999995,"15min":11.996},"last":0.751},{"interface":"front","average":{"1min":1.6419999999999999,"5min":1.3029999999999999,"15min":1.361},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.33400000000000002},"max":{"1min":7.4489999999999998,"5min":8.9169999999999998,"15min":13.634},"last":5.3899999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0109999999999999,"15min":1.159},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.1920000000000002,"5min":8.9260000000000002,"15min":9.9510000000000005},"last":5.3449999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.591,"15min":1.3999999999999999},"min":{"1min":0.39300000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3889999999999993,"5min":8.3889999999999993,"15min":8.6199999999999992},"last":0.76100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626786,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":321796,"kb_used_data":20476,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93423356,"statfs":{"total":95995035648,"available":95665516544,"internally_reserved":0,"allocated":20967424,"data_stored":20452888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.0669999999999999,"15min":1.1930000000000001},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.4359999999999999,"5min":8.2520000000000007,"15min":32.289999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.26,"5min":1.365,"15min":1.244},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":4.774,"5min":5.3109999999999999,"15min":31.454999999999998},"last":5.2519999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:27:19 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.367,"15min":1.617},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.4589999999999996,"5min":10.331,"15min":137.93899999999999},"last":2.4969999999999999},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.393,"15min":1.6439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.1790000000000003,"5min":8.2050000000000001,"15min":138.02500000000001},"last":2.5070000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:26:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.6000000000000001,"15min":1.786},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":4.6470000000000002,"5min":10.801,"15min":95.540000000000006},"last":3.0259999999999998},{"interface":"front","average":{"1min":1.649,"5min":1.526,"15min":1.772},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.8029999999999999,"5min":6.8250000000000002,"15min":96.352000000000004},"last":1.5620000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:26:53 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":2.2189999999999999,"15min":1.4910000000000001},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.327,"5min":23.315000000000001,"15min":23.315000000000001},"last":4.2229999999999999},{"interface":"front","average":{"1min":2.8159999999999998,"5min":1.819,"15min":1.28},"min":{"1min":0.40100000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.7430000000000003,"5min":23.274999999999999,"15min":23.274999999999999},"last":4.3650000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:27:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.3400000000000001,"15min":1.2809999999999999},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.26000000000000001},"max":{"1min":7.3090000000000002,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.603,"5min":1.524,"15min":1.4039999999999999},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.9379999999999997,"5min":6.9359999999999999,"15min":10.300000000000001},"last":2.532}]},{"osd":5,"last update":"Mon Oct 9 06:27:19 2023","interfaces":[{"interface":"back","average":{"1min":2.629,"5min":1.6359999999999999,"15min":1.492},"min":{"1min":0.36499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.7919999999999998,"5min":10.138999999999999,"15min":11.754},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.621,"5min":1.796,"15min":1.538},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.308},"max":{"1min":8.3249999999999993,"5min":11.026999999999999,"15min":12.295999999999999},"last":2.5760000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:26:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":1.7110000000000001,"15min":1.46},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":11.076000000000001,"5min":11.076000000000001,"15min":11.076000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.27,"5min":1.504,"15min":1.4350000000000001},"min":{"1min":0.46200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":4.2450000000000001,"5min":11.102,"15min":11.102},"last":0.58099999999999996}]}]},{"osd":1,"up_from":93,"seq":399431958696,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":195772,"kb_used_data":1652,"kb_used_omap":7,"kb_used_meta":194104,"kb_avail":93549380,"statfs":{"total":95995035648,"available":95794565120,"internally_reserved":0,"allocated":1691648,"data_stored":1010841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":198762896},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.282,"15min":1.137},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":8.2710000000000008,"5min":17.059000000000001,"15min":31.969000000000001},"last":0.62},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.454,"15min":1.3340000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.1010000000000009,"5min":14.243,"15min":40.667999999999999},"last":0.69899999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.6919999999999999,"15min":1.1899999999999999},"min":{"1min":0.25900000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.32,"5min":12.32,"15min":12.32},"last":1.1200000000000001},{"interface":"front","average":{"1min":3.1389999999999998,"5min":1.738,"15min":1.2569999999999999},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.542000000000002,"5min":17.542000000000002,"15min":17.542000000000002},"last":0.66600000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6309999999999998,"5min":5.194,"15min":2.4300000000000002},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":13.398999999999999,"5min":127.44499999999999,"15min":127.44499999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.9410000000000001,"5min":5.3120000000000003,"15min":2.4020000000000001},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.23200000000000001},"max":{"1min":8.2949999999999999,"5min":123.959,"15min":123.959},"last":0.73499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.5049999999999999,"15min":1.2949999999999999},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.3450000000000006,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":1.0429999999999999},{"interface":"front","average":{"1min":2.085,"5min":1.627,"15min":1.393},"min":{"1min":0.34799999999999998,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":8.1950000000000003,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.86499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.498,"15min":1.429},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":7.9400000000000004,"5min":14.170999999999999,"15min":14.170999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.6970000000000001,"15min":1.48},"min":{"1min":0.372,"5min":0.372,"15min":0.36599999999999999},"max":{"1min":8.3970000000000002,"5min":9.2810000000000006,"15min":9.2810000000000006},"last":1.071}]},{"osd":6,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.593,"15min":1.3440000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.36699999999999999},"max":{"1min":7.8550000000000004,"5min":14.212999999999999,"15min":14.212999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":2.298,"5min":1.7869999999999999,"15min":1.4610000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":9.1660000000000004,"5min":9.5790000000000006,"15min":9.5790000000000006},"last":1.097}]},{"osd":7,"last update":"Mon Oct 9 06:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":2.105,"15min":1.702},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.3759999999999994,"5min":30.774999999999999,"15min":30.774999999999999},"last":1.008},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.974,"15min":1.55},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.2279999999999998,"5min":30.815000000000001,"15min":30.815000000000001},"last":1.2030000000000001}]}]},{"osd":0,"up_from":88,"seq":377957122222,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270796,"kb_used_data":6340,"kb_used_omap":8,"kb_used_meta":264439,"kb_avail":93474356,"statfs":{"total":95995035648,"available":95717740544,"internally_reserved":0,"allocated":6492160,"data_stored":5796169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":270785792},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:27:10 2023","interfaces":[{"interface":"back","average":{"1min":4.0199999999999996,"5min":3.5600000000000001,"15min":1.9159999999999999},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":28.661000000000001,"5min":138.31700000000001,"15min":138.31700000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.4580000000000002,"5min":3.4159999999999999,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.952999999999999,"5min":144.37,"15min":144.37},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:27:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.841,"15min":1.345},"min":{"1min":0.27300000000000002,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":5.0190000000000001,"5min":28.745000000000001,"15min":28.745000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.867,"5min":1.9710000000000001,"15min":1.48},"min":{"1min":0.28299999999999997,"5min":0.22900000000000001,"15min":0.20200000000000001},"max":{"1min":12.292999999999999,"5min":28.146999999999998,"15min":28.146999999999998},"last":0.67600000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":2.544,"15min":1.476},"min":{"1min":0.23000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.1289999999999996,"5min":59.643000000000001,"15min":59.643000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.538,"5min":2.4279999999999999,"15min":1.4159999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2060000000000004,"5min":59.762999999999998,"15min":59.762999999999998},"last":0.61399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:26:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.302,"15min":1.3109999999999999},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1100000000000003,"5min":8.8729999999999993,"15min":9.0890000000000004},"last":0.85399999999999998},{"interface":"front","average":{"1min":2.6890000000000001,"5min":1.8009999999999999,"15min":1.548},"min":{"1min":0.441,"5min":0.40300000000000002,"15min":0.37},"max":{"1min":28.629000000000001,"5min":28.629000000000001,"15min":28.629000000000001},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:26:55 2023","interfaces":[{"interface":"back","average":{"1min":3.54,"5min":2.266,"15min":1.7470000000000001},"min":{"1min":0.436,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":28.574000000000002,"5min":28.574000000000002,"15min":28.574000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":3.0419999999999998,"5min":1.736,"15min":1.4790000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":28.061,"5min":28.061,"15min":28.061},"last":0.81799999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:26:55 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":1.762,"15min":1.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.34599999999999997},"max":{"1min":27.966999999999999,"5min":27.966999999999999,"15min":27.966999999999999},"last":0.752},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.2649999999999999,"15min":1.2490000000000001},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.1360000000000001,"5min":8.7629999999999999,"15min":8.7629999999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:26:55 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":1.9490000000000001,"15min":1.538},"min":{"1min":0.436,"5min":0.41499999999999998,"15min":0.38200000000000001},"max":{"1min":28.109999999999999,"5min":28.109999999999999,"15min":28.109999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.9769999999999999,"5min":1.8440000000000001,"15min":1.579},"min":{"1min":0.39600000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":28.074999999999999,"5min":28.074999999999999,"15min":28.074999999999999},"last":0.64500000000000002}]}]},{"osd":2,"up_from":98,"seq":420906795170,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313912,"kb_used_data":5744,"kb_used_omap":11,"kb_used_meta":308148,"kb_avail":93431240,"statfs":{"total":95995035648,"available":95673589760,"internally_reserved":0,"allocated":5881856,"data_stored":5201072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":315544510},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.9239999999999999,"15min":1.4670000000000001},"min":{"1min":0.245,"5min":0.19900000000000001,"15min":0.17899999999999999},"max":{"1min":6.0229999999999997,"5min":10.404,"15min":35.015999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":2.452,"5min":1.9810000000000001,"15min":1.6299999999999999},"min":{"1min":0.22800000000000001,"5min":0.182,"15min":0.182},"max":{"1min":8.6289999999999996,"5min":11.983000000000001,"15min":36.415999999999997},"last":1.958}]},{"osd":1,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":1.927,"15min":1.278},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20899999999999999},"max":{"1min":7.1630000000000003,"5min":11.571,"15min":11.571},"last":3.2080000000000002},{"interface":"front","average":{"1min":1.976,"5min":2.242,"15min":1.579},"min":{"1min":0.23699999999999999,"5min":0.23400000000000001,"15min":0.23100000000000001},"max":{"1min":9.2599999999999998,"5min":11.943,"15min":11.943},"last":1.986}]},{"osd":3,"last update":"Mon Oct 9 06:27:03 2023","interfaces":[{"interface":"back","average":{"1min":2.629,"5min":2.1619999999999999,"15min":1.3680000000000001},"min":{"1min":0.36599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.861000000000001,"5min":14.035,"15min":14.035},"last":5.8390000000000004},{"interface":"front","average":{"1min":2.665,"5min":2.4089999999999998,"15min":1.4219999999999999},"min":{"1min":0.26600000000000001,"5min":0.21299999999999999,"15min":0.21199999999999999},"max":{"1min":11.438000000000001,"5min":11.438000000000001,"15min":18.831},"last":3.0539999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":2.0550000000000002,"15min":1.796},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.9249999999999998,"5min":12.021000000000001,"15min":19.184000000000001},"last":1.9159999999999999},{"interface":"front","average":{"1min":2.0089999999999999,"5min":2.1629999999999998,"15min":1.6200000000000001},"min":{"1min":0.36399999999999999,"5min":0.34200000000000003,"15min":0.33800000000000002},"max":{"1min":6.2169999999999996,"5min":29.408000000000001,"15min":29.408000000000001},"last":3.1339999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":2.4289999999999998,"15min":1.8100000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.31},"max":{"1min":10.585000000000001,"5min":12.08,"15min":12.833},"last":1.9370000000000001},{"interface":"front","average":{"1min":2.0089999999999999,"5min":2.0649999999999999,"15min":1.5289999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.1740000000000004,"5min":29.545000000000002,"15min":29.545000000000002},"last":3.165}]},{"osd":6,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":1.9390000000000001,"15min":1.4019999999999999},"min":{"1min":0.38700000000000001,"5min":0.374,"15min":0.372},"max":{"1min":6.2409999999999997,"5min":29.504000000000001,"15min":29.504000000000001},"last":3.181},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.8480000000000001,"15min":1.4530000000000001},"min":{"1min":0.41699999999999998,"5min":0.38400000000000001,"15min":0.33600000000000002},"max":{"1min":7.077,"5min":7.0970000000000004,"15min":8.1349999999999998},"last":2.8889999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4220000000000002,"5min":1.923,"15min":1.546},"min":{"1min":0.45500000000000002,"5min":0.40500000000000003,"15min":0.308},"max":{"1min":7.1390000000000002,"5min":7.165,"15min":8.3309999999999995},"last":2.8570000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":2.2679999999999998,"15min":1.8280000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.141,"5min":12.051,"15min":12.051},"last":1.8859999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631644,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":291828,"kb_used_data":5740,"kb_used_omap":3,"kb_used_meta":286076,"kb_avail":93453324,"statfs":{"total":95995035648,"available":95696203776,"internally_reserved":0,"allocated":5877760,"data_stored":5200631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":292942369},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":3.1579999999999999,"15min":1.9410000000000001},"min":{"1min":0.215,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":12.913,"5min":147.988,"15min":147.988},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.8500000000000001,"5min":2.9220000000000002,"15min":1.883},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.215},"max":{"1min":8.8689999999999998,"5min":147.94200000000001,"15min":147.94200000000001},"last":0.78500000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.587,"15min":1.0589999999999999},"min":{"1min":0.21099999999999999,"5min":0.191,"15min":0.191},"max":{"1min":4.4210000000000003,"5min":10.564,"15min":10.564},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.8300000000000001,"15min":1.22},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.9550000000000001,"5min":15.119,"15min":15.119},"last":0.82899999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":1.9119999999999999,"15min":1.6639999999999999},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.951000000000001,"5min":12.951000000000001,"15min":70.594999999999999},"last":0.91700000000000004},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.411},"min":{"1min":0.40000000000000002,"5min":0.25900000000000001,"15min":0.23499999999999999},"max":{"1min":5.0910000000000002,"5min":8.7940000000000005,"15min":62.991999999999997},"last":0.94099999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.476,"15min":1.3819999999999999},"min":{"1min":0.46700000000000003,"5min":0.35399999999999998,"15min":0.314},"max":{"1min":2.694,"5min":8.3780000000000001,"15min":10.446999999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.994,"15min":1.5089999999999999},"min":{"1min":0.33700000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.5489999999999995,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.77000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.879,"15min":1.4139999999999999},"min":{"1min":0.35899999999999999,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":10.196999999999999,"5min":11.669,"15min":11.669},"last":0.86599999999999999},{"interface":"front","average":{"1min":2.149,"5min":1.6850000000000001,"15min":1.504},"min":{"1min":0.42999999999999999,"5min":0.38600000000000001,"15min":0.34300000000000003},"max":{"1min":8.9280000000000008,"5min":11.279,"15min":11.754},"last":1.0189999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.825,"15min":1.355},"min":{"1min":0.51700000000000002,"5min":0.36899999999999999,"15min":0.36099999999999999},"max":{"1min":6.5069999999999997,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":0.996},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.6020000000000001,"15min":1.3260000000000001},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.673,"5min":10.428000000000001,"15min":10.428000000000001},"last":1.0509999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":1.9119999999999999,"15min":1.452},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39400000000000002},"max":{"1min":18.695,"5min":18.695,"15min":18.695},"last":1.0840000000000001},{"interface":"front","average":{"1min":2.5710000000000002,"5min":2.0369999999999999,"15min":1.5469999999999999},"min":{"1min":0.55000000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":7.351,"5min":12.193,"15min":12.193},"last":1.1020000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019252,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317964,"kb_used_data":20740,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93427188,"statfs":{"total":95995035648,"available":95669440512,"internally_reserved":0,"allocated":21237760,"data_stored":20715175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.393,"15min":1.921},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.2400000000000002,"5min":9.3949999999999996,"15min":84.888000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.4900000000000002,"5min":1.994,"15min":2.2109999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":28.978999999999999,"5min":28.978999999999999,"15min":84.912999999999997},"last":0.33300000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.016,"5min":2.9350000000000001,"15min":2.0939999999999999},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.23999999999999999},"max":{"1min":7.3319999999999999,"5min":179.47300000000001,"15min":179.47300000000001},"last":4.601},{"interface":"front","average":{"1min":2.6970000000000001,"5min":3.3460000000000001,"15min":2.4260000000000002},"min":{"1min":0.31900000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5220000000000002,"5min":179.447,"15min":179.447},"last":4.6159999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:26:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3300000000000001,"5min":1.5700000000000001,"15min":1.857},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.0899999999999999,"5min":7.9489999999999998,"15min":61.783999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.4940000000000002,"5min":1.8280000000000001,"15min":1.9570000000000001},"min":{"1min":0.34300000000000003,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.234,"5min":10.56,"15min":61.551000000000002},"last":2.827}]},{"osd":3,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":2.7719999999999998,"15min":2},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.33000000000000002},"max":{"1min":30.52,"5min":91.361999999999995,"15min":91.361999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.6259999999999999,"5min":2.464,"15min":1.714},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.313},"max":{"1min":30.452000000000002,"5min":89.676000000000002,"15min":89.676000000000002},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:27:18 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.196,"15min":1.329},"min":{"1min":0.35799999999999998,"5min":0.309,"15min":0.309},"max":{"1min":8.8369999999999997,"5min":8.8369999999999997,"15min":12.242000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.431,"15min":1.6619999999999999},"min":{"1min":0.33600000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.589,"5min":8.1349999999999998,"15min":20.113},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 06:26:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.4690000000000001,"15min":1.5069999999999999},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.29499999999999998},"max":{"1min":4.0119999999999996,"5min":7.8529999999999998,"15min":18.010000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.23,"15min":1.3240000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.505000000000001,"5min":11.505000000000001,"15min":13.913},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.883,"15min":1.6699999999999999},"min":{"1min":0.41799999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.2789999999999999,"5min":7.8840000000000003,"15min":15.294},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.9530000000000001,"15min":1.6799999999999999},"min":{"1min":0.499,"5min":0.37,"15min":0.37},"max":{"1min":10.734,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.41299999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823019,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310716,"kb_used_data":9396,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93434436,"statfs":{"total":95995035648,"available":95676862464,"internally_reserved":0,"allocated":9621504,"data_stored":9110460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.988,"15min":2.0920000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.247},"max":{"1min":5.069,"5min":45.198999999999998,"15min":69.793999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.76,"15min":1.8280000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.3259999999999996,"5min":44.470999999999997,"15min":68.873000000000005},"last":1.4239999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:27:22 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":3.4060000000000001,"15min":2.2610000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.3920000000000003,"5min":161.71199999999999,"15min":161.71199999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":3.1760000000000002,"5min":3.653,"15min":2.4430000000000001},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.29399999999999998},"max":{"1min":13.093999999999999,"5min":162.50200000000001,"15min":162.50200000000001},"last":2.5800000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.1469999999999998,"15min":1.825},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.9089999999999998,"5min":45.466000000000001,"15min":47.207000000000001},"last":7.5270000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.97,"15min":1.802},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30499999999999999},"max":{"1min":4.5,"5min":59.082999999999998,"15min":59.082999999999998},"last":0.32500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":2.7450000000000001,"15min":1.8360000000000001},"min":{"1min":0.439,"5min":0.38100000000000001,"15min":0.29199999999999998},"max":{"1min":6.3200000000000003,"5min":66.153999999999996,"15min":66.153999999999996},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.744,"5min":2.4569999999999999,"15min":1.6890000000000001},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6970000000000001,"5min":65.792000000000002,"15min":65.792000000000002},"last":0.58199999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:27:22 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":1.7470000000000001,"15min":1.7789999999999999},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":8.5459999999999994,"5min":8.5459999999999994,"15min":17.667999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.633,"5min":1.538,"15min":1.5620000000000001},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.2450000000000001,"5min":7.3410000000000002,"15min":16.181999999999999},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":1.75,"15min":1.605},"min":{"1min":0.374,"5min":0.372,"15min":0.35699999999999998},"max":{"1min":8.5099999999999998,"5min":15.494999999999999,"15min":17.454999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.274,"5min":1.6639999999999999,"15min":1.585},"min":{"1min":0.50800000000000001,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":8.9009999999999998,"5min":9.2919999999999998,"15min":12.365},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:26:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":1.7849999999999999,"15min":1.6759999999999999},"min":{"1min":0.51300000000000001,"5min":0.48799999999999999,"15min":0.41699999999999998},"max":{"1min":6.9160000000000004,"5min":9.3170000000000002,"15min":13.047000000000001},"last":1.117},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.4690000000000001,"15min":1.506},"min":{"1min":0.47099999999999997,"5min":0.40899999999999997,"15min":0.36799999999999999},"max":{"1min":3.9729999999999999,"5min":10.465999999999999,"15min":11.981},"last":0.38700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":7340032,"data_stored":7340032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":7147520,"data_stored":7143448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":5177344,"data_stored":5177344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":9502720,"data_stored":9502720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:27:36.097 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:27:36.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-09T06:27:36.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:36.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:36.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:36.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:36 smithi155 bash[59495]: audit 2023-10-09T06:27:34.752279+0000 mgr.y (mgr.25723) 679 : audit 0 from='client.35196 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:36 smithi155 bash[59495]: cluster 2023-10-09T06:27:34.980326+0000 mgr.y (mgr.25723) 680 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 466 op/s 2023-10-09T06:27:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:36 smithi155 bash[59495]: cluster 2023-10-09T06:27:35.310661+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:36 smithi155 bash[59495]: cluster 2023-10-09T06:27:35.580750+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e157 with expected crc 2023-10-09T06:27:36.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:36.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:36.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:36.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:36.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:36.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:36.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:36.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:36.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:36.880 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:36.880 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:36.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:36.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:36.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:37.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:37.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:37.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:37.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:37.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:37.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:27:37.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:37.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:37.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:37.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:37.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:37.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:37.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:38.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:38.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:38.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:38.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:38.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:38.286 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:38 smithi155 bash[59495]: cluster 2023-10-09T06:27:36.980941+0000 mgr.y (mgr.25723) 681 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-10-09T06:27:38.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:38 smithi155 bash[59495]: audit 2023-10-09T06:27:37.112715+0000 mon.a (mon.0) 866 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:38 smithi155 bash[59495]: audit 2023-10-09T06:27:37.119713+0000 mon.a (mon.0) 867 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:38 smithi155 bash[59495]: audit 2023-10-09T06:27:37.972569+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:38 smithi155 bash[59495]: audit 2023-10-09T06:27:37.978769+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:38.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[77299]: cluster 2023-10-09T06:27:36.980941+0000 mgr.y (mgr.25723) 681 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-10-09T06:27:38.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[77299]: audit 2023-10-09T06:27:37.112715+0000 mon.a (mon.0) 866 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[77299]: audit 2023-10-09T06:27:37.119713+0000 mon.a (mon.0) 867 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[77299]: audit 2023-10-09T06:27:37.972569+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[77299]: audit 2023-10-09T06:27:37.978769+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[80002]: cluster 2023-10-09T06:27:36.980941+0000 mgr.y (mgr.25723) 681 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 356 op/s 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[80002]: audit 2023-10-09T06:27:37.112715+0000 mon.a (mon.0) 866 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[80002]: audit 2023-10-09T06:27:37.119713+0000 mon.a (mon.0) 867 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[80002]: audit 2023-10-09T06:27:37.972569+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:38 smithi125 bash[80002]: audit 2023-10-09T06:27:37.978769+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:38.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:38.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:38.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:38.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:38.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:38.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:38.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:38.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:38.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:38.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:38.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:39.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:39.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:39.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:39.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:39.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:39.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[77299]: audit 2023-10-09T06:27:38.449039+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:39.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[77299]: audit 2023-10-09T06:27:38.450641+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:39.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[77299]: audit 2023-10-09T06:27:38.458589+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:39.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[80002]: audit 2023-10-09T06:27:38.449039+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:39.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[80002]: audit 2023-10-09T06:27:38.450641+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:39.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:39 smithi125 bash[80002]: audit 2023-10-09T06:27:38.458589+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:39.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:39.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:39 smithi155 bash[59495]: audit 2023-10-09T06:27:38.449039+0000 mon.a (mon.0) 870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:39.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:39 smithi155 bash[59495]: audit 2023-10-09T06:27:38.450641+0000 mon.a (mon.0) 871 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:39.349 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:39 smithi155 bash[59495]: audit 2023-10-09T06:27:38.458589+0000 mon.a (mon.0) 872 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:27:39.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:39.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:39.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:39.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:39.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:39.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:39.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:39.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:39.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:39.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:39.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:39.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:39.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:40.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:40.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:40 smithi155 bash[59495]: cluster 2023-10-09T06:27:38.981757+0000 mgr.y (mgr.25723) 682 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 288 op/s 2023-10-09T06:27:40.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:40.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:40.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:40.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:40 smithi125 bash[80002]: cluster 2023-10-09T06:27:38.981757+0000 mgr.y (mgr.25723) 682 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 288 op/s 2023-10-09T06:27:40.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:40 smithi125 bash[77299]: cluster 2023-10-09T06:27:38.981757+0000 mgr.y (mgr.25723) 682 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 288 op/s 2023-10-09T06:27:40.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:40.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:40.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:40.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:40.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:40.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.815 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:40.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:40.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:40.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:41.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:41.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:41.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:41.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:41.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:41.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:41.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:41.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:41.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:41.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:41.628 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:41] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.33.4" 2023-10-09T06:27:41.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:41.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:41.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:41.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:41.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:41.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:42.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:42.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:42.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:42.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:42.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:42.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:42.343 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:42 smithi125 bash[77299]: cluster 2023-10-09T06:27:40.982744+0000 mgr.y (mgr.25723) 683 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 117 op/s 2023-10-09T06:27:42.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:42 smithi125 bash[80002]: cluster 2023-10-09T06:27:40.982744+0000 mgr.y (mgr.25723) 683 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 117 op/s 2023-10-09T06:27:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:42 smithi155 bash[59495]: cluster 2023-10-09T06:27:40.982744+0000 mgr.y (mgr.25723) 683 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 117 op/s 2023-10-09T06:27:42.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:42.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:42.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:42.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:42.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:42.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:42.725 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.737 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76727 5.0 5:09f0aaf0:::benchmark_data_smithi125_106927_object76726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.150319+0000", 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 521.57872945400004, 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21574062899999999, 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76727 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.738 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150319+0000", 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150319+0000", 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150318+0000", 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150342+0000", 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4202e-05 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150357+0000", 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4061e-05 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.150361+0000", 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8160000000000001e-06 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.152713+0000", 2023-10-09T06:27:42.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023520519999999999 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.152829+0000", 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011601500000000001 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.152944+0000", 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000114226 2023-10-09T06:27:42.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.156958+0000", 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040146230000000001 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158902+0000", 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00194341 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.365785+0000", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20688352600000001 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.365951+0000", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016549699999999999 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.366059+0000", 2023-10-09T06:27:42.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010880899999999999 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76729 5.0 5:0af4ee0c:::benchmark_data_smithi125_106927_object76728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.153639+0000", 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 521.57540945799997, 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21375629700000001, 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.745 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76729 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153639+0000", 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153639+0000", 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153638+0000", 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153663+0000", 2023-10-09T06:27:42.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5077999999999999e-05 2023-10-09T06:27:42.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153677+0000", 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4409999999999999e-05 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.153685+0000", 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1159999999999999e-06 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.154782+0000", 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010968429999999999 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.154872+0000", 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9826000000000004e-05 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.154965+0000", 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2590999999999995e-05 2023-10-09T06:27:42.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.156969+0000", 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00200395 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158938+0000", 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019690929999999999 2023-10-09T06:27:42.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.367044+0000", 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20810614799999999 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.367344+0000", 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030038299999999999 2023-10-09T06:27:42.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.367395+0000", 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1004000000000002e-05 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44693.0:76733 5.0 5:07f0bf0e:::benchmark_data_smithi125_106927_object76732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:27:42.754 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:19:01.158562+0000", 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 521.57048620299997, 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.223924074, 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44693", 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 76733 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158562+0000", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158562+0000", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158561+0000", 2023-10-09T06:27:42.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158586+0000", 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5817999999999999e-05 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158601+0000", 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4734e-05 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158611+0000", 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9450000000000005e-06 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.158964+0000", 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035256799999999999 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.159114+0000", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015031800000000001 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.159272+0000", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015808200000000001 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.162392+0000", 2023-10-09T06:27:42.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031202880000000001 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.164650+0000", 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022573820000000001 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.382385+0000", 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21773498299999999 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.382439+0000", 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4410999999999998e-05 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:19:01.382486+0000", 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7116000000000001e-05 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.761 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39495 6.c 6:37822acc:::benchmark_data_smithi125_130224_object39494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.946777+0000", 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.782271165, 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.218222365, 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.762 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39495 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946777+0000", 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946777+0000", 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946776+0000", 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946796+0000", 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9718e-05 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946805+0000", 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.4599999999999992e-06 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.946816+0000", 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.082e-05 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.948771+0000", 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019546289999999998 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.948872+0000", 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100717 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.950873+0000", 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020014289999999999 2023-10-09T06:27:42.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.954749+0000", 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038765420000000002 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958999+0000", 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004249109 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164914+0000", 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20591525599999999 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164960+0000", 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5828999999999998e-05 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165000+0000", 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9836000000000003e-05 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39502 6.6 6:6c1b53bd:::benchmark_data_smithi125_130224_object39501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.952847+0000", 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77620153800001, 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20921189700000001, 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.769 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39502 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952847+0000", 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952847+0000", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952846+0000", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952864+0000", 2023-10-09T06:27:42.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8304999999999998e-05 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952874+0000", 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6909999999999992e-06 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952880+0000", 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3459999999999999e-06 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956122+0000", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003242319 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956271+0000", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014854700000000001 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956411+0000", 2023-10-09T06:27:42.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000140364 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958088+0000", 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001676325 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161886+0000", 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20379871599999999 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161960+0000", 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3758999999999996e-05 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162012+0000", 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2018999999999998e-05 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162059+0000", 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6502999999999998e-05 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39504 6.c 6:3bbba8c2:::benchmark_data_smithi125_130224_object39503:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.952895+0000", 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77615368900001, 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.212500982, 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.776 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39504 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952895+0000", 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952895+0000", 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952894+0000", 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952913+0000", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8734999999999999e-05 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952917+0000", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7350000000000004e-06 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952921+0000", 2023-10-09T06:27:42.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.23e-06 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.954772+0000", 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018518709999999999 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.954916+0000", 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014322299999999999 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.955070+0000", 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015438499999999999 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960461+0000", 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053912930000000001 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165038+0000", 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20457666999999999 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165300+0000", 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000261599 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165354+0000", 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3931999999999998e-05 2023-10-09T06:27:42.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165396+0000", 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2046999999999999e-05 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39505 6.c 6:3777c468:::benchmark_data_smithi125_130224_object39504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.783 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.952924+0000", 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.776124146, 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21509427, 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39505 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952924+0000", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952924+0000", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952924+0000", 2023-10-09T06:27:42.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952939+0000", 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4769e-05 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952942+0000", 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7400000000000002e-06 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.952945+0000", 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6010000000000002e-06 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956702+0000", 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0037565179999999999 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956837+0000", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013594200000000001 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957002+0000", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016499799999999999 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164229+0000", 2023-10-09T06:27:42.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20722701800000001 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165067+0000", 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00083799500000000002 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.167954+0000", 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002886193 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.167992+0000", 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8158000000000003e-05 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.168018+0000", 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6610000000000001e-05 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39507 6.c 6:37eb8a41:::benchmark_data_smithi125_130224_object39506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.953811+0000", 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77523708800001, 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21426953000000001, 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.791 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39507 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953811+0000", 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953811+0000", 2023-10-09T06:27:42.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953811+0000", 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953829+0000", 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8553e-05 2023-10-09T06:27:42.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953839+0000", 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.465e-06 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.953844+0000", 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6239999999999999e-06 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959044+0000", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.005200194 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959145+0000", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010097700000000001 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959233+0000", 2023-10-09T06:27:42.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7189999999999997e-05 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164840+0000", 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20560761899999999 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165097+0000", 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000257154 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.168026+0000", 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002928866 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.168051+0000", 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4655999999999999e-05 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.168081+0000", 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9870999999999999e-05 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44939.0:39504 6.c e146/142 ondisk, result = 0)", 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.956907+0000", 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77214081899999, 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20855219799999999, 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.798 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956907+0000", 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956907+0000", 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956907+0000", 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956911+0000", 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3159999999999998e-06 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956918+0000", 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.917e-06 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.956923+0000", 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0969999999999996e-06 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960447+0000", 2023-10-09T06:27:42.801 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035240950000000001 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960455+0000", 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.994e-06 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165460+0000", 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20500442699999999 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.802 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:39509 6.7 e146/142 6:ecb54e18:::benchmark_data_smithi125_130224_object39508:head v 146'2460, mlcod=146'2457)", 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.957255+0000", 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.771793352, 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20451831100000001, 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.803 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957255+0000", 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957255+0000", 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.804 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957254+0000", 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957334+0000", 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9472000000000001e-05 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957343+0000", 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8319999999999995e-06 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.805 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957349+0000", 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6649999999999997e-06 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957424+0000", 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4499000000000006e-05 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.806 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.957441+0000", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7652999999999999e-05 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.961018+0000", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003576217 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161773+0000", 2023-10-09T06:27:42.807 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20075557699999999 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39510 6.c 6:3454d73c:::benchmark_data_smithi125_130224_object39509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.958313+0000", 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.770735696, 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21100712499999999, 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.808 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39510 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958313+0000", 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.809 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958313+0000", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958311+0000", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958332+0000", 2023-10-09T06:27:42.810 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0108000000000001e-05 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958342+0000", 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0045e-05 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958347+0000", 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4949999999999999e-06 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.811 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960519+0000", 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021719560000000001 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960642+0000", 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012339600000000001 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.812 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960734+0000", 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1626999999999995e-05 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164886+0000", 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20415241200000001 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165126+0000", 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023970400000000001 2023-10-09T06:27:42.813 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.168091+0000", 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029648669999999999 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.169280+0000", 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001188845 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.814 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.169320+0000", 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9817000000000002e-05 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39511 6.6 6:68b4bb9b:::benchmark_data_smithi125_130224_object39510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.958353+0000", 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77069535699999, 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20603079899999999, 2023-10-09T06:27:42.815 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39511 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958353+0000", 2023-10-09T06:27:42.816 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958353+0000", 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958353+0000", 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.817 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958368+0000", 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5690000000000001e-05 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958372+0000", 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9650000000000002e-06 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.818 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958376+0000", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2880000000000001e-06 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.958414+0000", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8436999999999998e-05 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960259+0000", 2023-10-09T06:27:42.819 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0018452189999999999 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960409+0000", 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015018799999999999 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162113+0000", 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.201703559 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.820 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162147+0000", 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3862000000000002e-05 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164294+0000", 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0021468770000000002 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.821 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164341+0000", 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7635e-05 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164384+0000", 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2463999999999999e-05 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.822 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:39513 6.7 e146/142 6:e0d16f4c:::benchmark_data_smithi125_130224_object39512:head v 146'2461, mlcod=146'2458)", 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.959009+0000", 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.77003937699999, 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20753245400000001, 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959009+0000", 2023-10-09T06:27:42.823 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959009+0000", 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959008+0000", 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.824 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959055+0000", 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6987999999999998e-05 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959065+0000", 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6180000000000001e-06 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.825 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959071+0000", 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6159999999999998e-06 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161956+0000", 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20288478099999999 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162086+0000", 2023-10-09T06:27:42.826 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012962600000000001 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.166475+0000", 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043890220000000002 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.166541+0000", 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6612999999999994e-05 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.827 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44939.0:39502 6.6 e146/142 ondisk, result = 0)", 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.959240+0000", 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.76980810000001, 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20265140400000001, 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.828 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959240+0000", 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959240+0000", 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.829 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959240+0000", 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959244+0000", 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.69e-06 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959252+0000", 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9109999999999993e-06 2023-10-09T06:27:42.830 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959258+0000", 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2330000000000003e-06 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161872+0000", 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20261304099999999 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.831 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161880+0000", 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1300000000000001e-06 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161892+0000", 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1975e-05 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.832 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39516 6.c 6:32f8f320:::benchmark_data_smithi125_130224_object39515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.959553+0000", 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.769495689, 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21429141900000001, 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39516 2023-10-09T06:27:42.833 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959553+0000", 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959553+0000", 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.834 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959552+0000", 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959571+0000", 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8808999999999999e-05 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.835 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959581+0000", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0043e-05 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.959589+0000", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165493+0000", 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.205904219 2023-10-09T06:27:42.836 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165645+0000", 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151855 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165808+0000", 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016291600000000001 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.837 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.169363+0000", 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035556139999999999 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.172335+0000", 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029718380000000001 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:27:42.838 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.173742+0000", 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001406572 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.173815+0000", 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3632000000000002e-05 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.173844+0000", 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8713000000000001e-05 2023-10-09T06:27:42.839 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:39514 6.8 e146/142 6:1216b66a:::benchmark_data_smithi125_130224_object39513:head v 146'2476, mlcod=146'2473)", 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.960143+0000", 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.76890577699999, 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20360455499999999, 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.840 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960143+0000", 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960143+0000", 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.841 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960142+0000", 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960174+0000", 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2397000000000002e-05 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960185+0000", 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.059e-05 2023-10-09T06:27:42.842 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960192+0000", 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0180000000000004e-06 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960236+0000", 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4462999999999998e-05 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.843 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.960256+0000", 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9405000000000001e-05 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.163691+0000", 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.203435068 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.844 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.163747+0000", 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6351000000000003e-05 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44939.0:39505 6.c e146/142 ondisk, result = 0)", 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.961401+0000", 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.76764716899999, 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20283119499999999, 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:27:42.845 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.961401+0000", 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.961401+0000", 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.846 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.961400+0000", 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.961405+0000", 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8960000000000002e-06 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.847 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161704+0000", 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20029857600000001 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161724+0000", 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.065e-05 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164213+0000", 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024887569999999999 2023-10-09T06:27:42.848 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164222+0000", 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0510000000000002e-06 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.164232+0000", 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0032e-05 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.849 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.44939.0:39502 6.6 e146/142 ondisk, result = 0)", 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.962187+0000", 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.76686162499999, 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.199884652, 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.850 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.962187+0000", 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.962187+0000", 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.962186+0000", 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.851 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.962190+0000", 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0609999999999997e-06 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161769+0000", 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19957918599999999 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.852 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161777+0000", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5229999999999998e-06 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161912+0000", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013558400000000001 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161921+0000", 2023-10-09T06:27:42.853 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3920000000000003e-06 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.162071+0000", 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015049700000000001 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.854 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:39518 6.8 e146/142 6:13d457a8:::benchmark_data_smithi125_130224_object39517:head v 146'2477, mlcod=146'2474)", 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.963700+0000", 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 238.76534839999999, 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20224278900000001, 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.963700+0000", 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.855 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.963700+0000", 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.963699+0000", 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.856 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.963723+0000", 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.404e-05 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161702+0000", 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19797858099999999 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161724+0000", 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2614000000000001e-05 2023-10-09T06:27:42.857 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161791+0000", 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6356e-05 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.161820+0000", 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9232999999999999e-05 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.858 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165388+0000", 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035681549999999999 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.165943+0000", 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00055445400000000001 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.859 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:117536 6.4 e149/142 6:233b0323:::benchmark_data_smithi125_130224_object6345:head v 149'7491, mlcod=149'7489)", 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:25:01.734819+0000", 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 160.994229117, 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19879466100000001, 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734819+0000", 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.860 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734819+0000", 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734818+0000", 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.861 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734827+0000", 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3090000000000001e-06 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734840+0000", 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2422999999999999e-05 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:27:42.862 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.734848+0000", 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0749999999999998e-06 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.735277+0000", 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00042921099999999998 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.735298+0000", 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1308999999999999e-05 2023-10-09T06:27:42.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.933493+0000", 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19819462900000001 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.933614+0000", 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120869 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.864 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:27:42.865 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:27:42.865 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:27:42.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:42.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:27:42.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:27:42.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:27:42.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:27:42.867 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:27:42.878 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:27:42.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:42.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:42.902 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:27:42.903 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.904 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:27:42.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:43.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:43.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:43.266 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:43 smithi125 bash[77299]: audit 2023-10-09T06:27:42.698339+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.125:0/4239156855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:43.267 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:43 smithi125 bash[80002]: audit 2023-10-09T06:27:42.698339+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.125:0/4239156855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:43.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:43.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:43.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:43 smithi155 bash[59495]: audit 2023-10-09T06:27:42.698339+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.125:0/4239156855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:43.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:43.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:43.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:43.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:43.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:43.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:43.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:43.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:44.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:44.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:44.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:44.169 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:44 smithi125 bash[77299]: cluster 2023-10-09T06:27:42.983490+0000 mgr.y (mgr.25723) 684 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 33 op/s 2023-10-09T06:27:44.169 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:44 smithi125 bash[80002]: cluster 2023-10-09T06:27:42.983490+0000 mgr.y (mgr.25723) 684 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 33 op/s 2023-10-09T06:27:44.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:44.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:44.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:44.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:44 smithi155 bash[59495]: cluster 2023-10-09T06:27:42.983490+0000 mgr.y (mgr.25723) 684 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 33 op/s 2023-10-09T06:27:44.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:44.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:44.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:44.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:44.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:44.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:44.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:44.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:44.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:44.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:44.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:45.000 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:45.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:45.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:45.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:45.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.407 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:45.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:45.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:45.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:45.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:27:45.779 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:45.803 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:45.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:45.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:45.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:46.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:46.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:46.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:46.174 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:46 smithi125 bash[77299]: cluster 2023-10-09T06:27:44.984493+0000 mgr.y (mgr.25723) 685 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s 2023-10-09T06:27:46.175 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:46 smithi125 bash[80002]: cluster 2023-10-09T06:27:44.984493+0000 mgr.y (mgr.25723) 685 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s 2023-10-09T06:27:46.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:46.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:46.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:46 smithi155 bash[59495]: cluster 2023-10-09T06:27:44.984493+0000 mgr.y (mgr.25723) 685 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 16 op/s 2023-10-09T06:27:46.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:46.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:46.481 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:46.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:46.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:46.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:46.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:27:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:46.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:46.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:46.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:46.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:46.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:47.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:47.112 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:47.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:47.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:47 smithi125 bash[77299]: audit 2023-10-09T06:27:46.715785+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.125:0/3653502453' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:47 smithi125 bash[77299]: audit 2023-10-09T06:27:46.716331+0000 mon.a (mon.0) 873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:47.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:47.376 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:47.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:47.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:47 smithi155 bash[59495]: audit 2023-10-09T06:27:46.715785+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.125:0/3653502453' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.492 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:47 smithi155 bash[59495]: audit 2023-10-09T06:27:46.716331+0000 mon.a (mon.0) 873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:47.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:47.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:47 smithi125 bash[80002]: audit 2023-10-09T06:27:46.715785+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.125:0/3653502453' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:47 smithi125 bash[80002]: audit 2023-10-09T06:27:46.716331+0000 mon.a (mon.0) 873 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:27:47.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:47.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:47.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:47.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:47.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:47.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:48.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:27:48.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:48.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:48.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:48.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:48.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:48.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:48.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:48 smithi125 bash[77299]: cluster 2023-10-09T06:27:46.985014+0000 mgr.y (mgr.25723) 686 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:48.441 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:48 smithi125 bash[80002]: cluster 2023-10-09T06:27:46.985014+0000 mgr.y (mgr.25723) 686 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:48.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:48.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:48.572 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:48 smithi155 bash[59495]: cluster 2023-10-09T06:27:46.985014+0000 mgr.y (mgr.25723) 686 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:48.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:27:48.649 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T06:27:48.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T06:27:48.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:48.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:48.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:48.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:48.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:49.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:49.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:49.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:49.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:49.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:49.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:49 smithi125 bash[77299]: audit 2023-10-09T06:27:48.579868+0000 mon.a (mon.0) 874 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:49.377 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:49 smithi125 bash[80002]: audit 2023-10-09T06:27:48.579868+0000 mon.a (mon.0) 874 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:49 smithi155 bash[59495]: audit 2023-10-09T06:27:48.579868+0000 mon.a (mon.0) 874 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:49.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:49.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:49.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:49.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:49.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:49.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:49.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:49.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:49.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:49.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:49.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:50.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:50.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:50.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:50.175 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:50 smithi125 bash[77299]: cluster 2023-10-09T06:27:48.985642+0000 mgr.y (mgr.25723) 687 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:50.175 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:50 smithi125 bash[80002]: cluster 2023-10-09T06:27:48.985642+0000 mgr.y (mgr.25723) 687 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:50.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:50.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:50.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:50.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:50.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:50.457 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:27:50.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:50.490 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:50 smithi155 bash[59495]: cluster 2023-10-09T06:27:48.985642+0000 mgr.y (mgr.25723) 687 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:50.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:50.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:50.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:50.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:50.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:50.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:50.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:50.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:50.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:51.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:51.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:51.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:51.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:51.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:51.381 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:27:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:27:51] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T06:27:51.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:51.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:51.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:51.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:51.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:51.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:51.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:51.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:51.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:51.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:52.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:52.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:52.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:52.173 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:27:52.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:52.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:52.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:52 smithi155 bash[59495]: cluster 2023-10-09T06:27:50.986625+0000 mgr.y (mgr.25723) 688 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:52.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:52 smithi155 bash[59495]: audit 2023-10-09T06:27:51.603355+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.125:0/411734976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:52 smithi155 bash[59495]: audit 2023-10-09T06:27:51.604087+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:52.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[77299]: cluster 2023-10-09T06:27:50.986625+0000 mgr.y (mgr.25723) 688 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:52.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[77299]: audit 2023-10-09T06:27:51.603355+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.125:0/411734976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[77299]: audit 2023-10-09T06:27:51.604087+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[80002]: cluster 2023-10-09T06:27:50.986625+0000 mgr.y (mgr.25723) 688 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:52.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[80002]: audit 2023-10-09T06:27:51.603355+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.125:0/411734976' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:52 smithi125 bash[80002]: audit 2023-10-09T06:27:51.604087+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:52.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:52.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:52.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:52.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:52.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:52.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:52.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:52.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:52.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:53.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:53.160 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.150223+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:53.160 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[77299]: audit 2023-10-09T06:27:52.162927+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:53.160 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.163026+0000 mon.a (mon.0) 878 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:27:53.161 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.477859+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.161 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.478904+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:53 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.150223+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:53.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:53 smithi155 bash[59495]: audit 2023-10-09T06:27:52.162927+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:53.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:53 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.163026+0000 mon.a (mon.0) 878 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:27:53.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:53 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.477859+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.237 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:53 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.478904+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:53.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:53.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.150223+0000 mon.a (mon.0) 876 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:53.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[80002]: audit 2023-10-09T06:27:52.162927+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:53.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.163026+0000 mon.a (mon.0) 878 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:27:53.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.477859+0000 osd.4 (osd.4) 83 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:53 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.478904+0000 osd.4 (osd.4) 84 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:53.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:53.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:53.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:53.586 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:53.587 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:53.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.650 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:53.650 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:53.650 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:53.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T06:27:53.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:53.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:53.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:53.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:53.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:53.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:54.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:54.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:54.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:54.183 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.885184+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.183 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.885972+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.183 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[80002]: cluster 2023-10-09T06:27:52.987215+0000 mgr.y (mgr.25723) 689 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:54.184 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.885184+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.184 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.885972+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.184 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:54 smithi125 bash[77299]: cluster 2023-10-09T06:27:52.987215+0000 mgr.y (mgr.25723) 689 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:54.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:54 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.885184+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:54 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.885972+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e158 with expected crc 2023-10-09T06:27:54.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:54 smithi155 bash[59495]: cluster 2023-10-09T06:27:52.987215+0000 mgr.y (mgr.25723) 689 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:54.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:54.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:54.447 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:54.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:54.568 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:54.569 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:54.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:54.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:54.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:54.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:54.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:54.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:54.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:55.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:27:55.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:55.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:55.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:55.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:55.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:55.379 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:27:55.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:55.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:55.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:55.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:55.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:55.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:55.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:55.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:55.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:55.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:56.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:56.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:56.135 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:56.159 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:56.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:56.275 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:56 smithi155 bash[59495]: cluster 2023-10-09T06:27:54.988203+0000 mgr.y (mgr.25723) 690 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:56.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:56.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:56.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:56 smithi125 bash[77299]: cluster 2023-10-09T06:27:54.988203+0000 mgr.y (mgr.25723) 690 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:56.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:56 smithi125 bash[80002]: cluster 2023-10-09T06:27:54.988203+0000 mgr.y (mgr.25723) 690 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:56.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:56.526 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:56.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:56.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:56.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.817 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:56.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:56.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:56.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:57.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:57.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:57.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:57.360 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:57.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:57.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:57.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:57.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:57.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:57.657 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:57.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:57.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:57.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:57.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:58.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:58.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:58.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:58.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:58.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:58.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:58.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:58 smithi155 bash[59495]: cluster 2023-10-09T06:27:56.988739+0000 mgr.y (mgr.25723) 691 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:58.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:58.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:58.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:58.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:58.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:58.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:58 smithi125 bash[77299]: cluster 2023-10-09T06:27:56.988739+0000 mgr.y (mgr.25723) 691 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:58.599 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:58 smithi125 bash[80002]: cluster 2023-10-09T06:27:56.988739+0000 mgr.y (mgr.25723) 691 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:27:58.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:58.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:58.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:58.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:58.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:58.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:58.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:58.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:59.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:59.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:59.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:59.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:59.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:59.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:59.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:59.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:59.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:59.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:59.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:59.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:59.689 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:59.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:59.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:59.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:59.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:59.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:59.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:59.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:27:59.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:00.007 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:27:59 smithi125 bash[77299]: cluster 2023-10-09T06:27:58.989349+0000 mgr.y (mgr.25723) 692 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:00.007 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:27:59 smithi125 bash[80002]: cluster 2023-10-09T06:27:58.989349+0000 mgr.y (mgr.25723) 692 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:00.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:00.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:27:59 smithi155 bash[59495]: cluster 2023-10-09T06:27:58.989349+0000 mgr.y (mgr.25723) 692 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:00.205 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:00.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:00.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:00.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:00.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:00.465 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:00.497 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:28:00.518 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:28:00.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:28:00.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:28:00.577 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:28:00.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.598 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:28:00.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:00.619 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:28:00.644 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:28:00.646 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.647 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:28:00.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:00.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:00.815 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_1' removed 2023-10-09T06:28:00.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:00.853 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:00 smithi125 bash[77299]: audit 2023-10-09T06:28:00.148185+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:28:00.853 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:00 smithi125 bash[77299]: audit 2023-10-09T06:28:00.464914+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.125:0/3623228895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:00.853 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:00 smithi125 bash[80002]: audit 2023-10-09T06:28:00.148185+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:28:00.854 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:00 smithi125 bash[80002]: audit 2023-10-09T06:28:00.464914+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.125:0/3623228895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:00.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:00.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:00.978 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:00 smithi155 bash[59495]: audit 2023-10-09T06:28:00.148185+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:28:00.978 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:00 smithi155 bash[59495]: audit 2023-10-09T06:28:00.464914+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.125:0/3623228895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:00.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:00.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:01.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:01.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:01.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:01.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:01.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:01.378 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:01] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-09T06:28:01.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:01.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:01.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:01.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:01.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:01.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:01.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:01.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:01.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:01.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:02.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:02.033 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: audit 2023-10-09T06:28:00.804548+0000 mon.a (mon.0) 880 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:28:02.033 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:00.804598+0000 mon.a (mon.0) 881 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:00.807925+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:00.989827+0000 mgr.y (mgr.25723) 693 : cluster 0 pgmap v669: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:01.086123+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:01.086369+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[77299]: cluster 2023-10-09T06:28:01.086588+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: audit 2023-10-09T06:28:00.804548+0000 mon.a (mon.0) 880 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:00.804598+0000 mon.a (mon.0) 881 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T06:28:02.034 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:00.807925+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:00.989827+0000 mgr.y (mgr.25723) 693 : cluster 0 pgmap v669: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:02.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:01.086123+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:01.086369+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:01 smithi125 bash[80002]: cluster 2023-10-09T06:28:01.086588+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: audit 2023-10-09T06:28:00.804548+0000 mon.a (mon.0) 880 : audit 1 from='client.? 172.21.15.125:0/477461671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:28:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:00.804598+0000 mon.a (mon.0) 881 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-09T06:28:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:00.807925+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:00.989827+0000 mgr.y (mgr.25723) 693 : cluster 0 pgmap v669: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:02.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:01.086123+0000 osd.4 (osd.4) 85 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:01.086369+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:01 smithi155 bash[59495]: cluster 2023-10-09T06:28:01.086588+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e159 with expected crc 2023-10-09T06:28:02.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:02.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:02.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:02.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:02.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:02.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:02.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:02.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:02.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:02.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:02.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:02.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:03.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:03.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:03.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:03.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:03.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:03.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:03.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:03.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:03.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:03.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:03.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:03.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:03.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:03.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:03.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:03.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:03.857 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:28:03.857 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:28:03.857 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:28:03.857 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:28:03.858 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:28:03.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:03.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:03.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:04.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:04.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:04.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:04.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:04.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:04.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:04.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:04 smithi125 bash[77299]: cluster 2023-10-09T06:28:02.990409+0000 mgr.y (mgr.25723) 694 : cluster 0 pgmap v670: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:04.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:04 smithi125 bash[77299]: audit 2023-10-09T06:28:03.580131+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:04.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:04 smithi125 bash[80002]: cluster 2023-10-09T06:28:02.990409+0000 mgr.y (mgr.25723) 694 : cluster 0 pgmap v670: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:04.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:04 smithi125 bash[80002]: audit 2023-10-09T06:28:03.580131+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:04 smithi155 bash[59495]: cluster 2023-10-09T06:28:02.990409+0000 mgr.y (mgr.25723) 694 : cluster 0 pgmap v670: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:04 smithi155 bash[59495]: audit 2023-10-09T06:28:03.580131+0000 mon.a (mon.0) 882 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:04.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:04.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:04.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:04.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:04.591 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:28:04.592 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:28:04.592 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T06:28:04.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T06:28:04.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:04.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:04.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:28:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:04.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:04.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:04.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:04.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:04.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:05.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:05.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:05.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:05.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:05.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:05.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:05.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:05.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:05.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:05.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:05.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:05.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:05.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:05.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:05.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:28:05.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:05.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:05.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:06.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:06.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:06.055 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:06 smithi125 bash[77299]: cluster 2023-10-09T06:28:04.990947+0000 mgr.y (mgr.25723) 695 : cluster 0 pgmap v671: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:06.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:06.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:06.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:06.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:06.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:06 smithi125 bash[80002]: cluster 2023-10-09T06:28:04.990947+0000 mgr.y (mgr.25723) 695 : cluster 0 pgmap v671: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:06.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:06 smithi155 bash[59495]: cluster 2023-10-09T06:28:04.990947+0000 mgr.y (mgr.25723) 695 : cluster 0 pgmap v671: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:06.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.425 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:06.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:06.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:06.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:06.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:06.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:06.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:06.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:06.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:07.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:07.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:07.221 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:07.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:07.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:07.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:07.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:07.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:07.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:07.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.877 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:07 smithi125 bash[77299]: debug 2023-10-09T06:28:07.749+0000 7ff949fd9700 -1 mon.a@0(leader).osd e159 definitely_dead 0 2023-10-09T06:28:07.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:07.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:08.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:08.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:08.081 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.6. 2023-10-09T06:28:08.131 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[77299]: cluster 2023-10-09T06:28:06.991683+0000 mgr.y (mgr.25723) 696 : cluster 0 pgmap v672: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:08.132 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[77299]: audit 2023-10-09T06:28:07.752220+0000 mon.a (mon.0) 883 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:28:08.132 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[77299]: cluster 2023-10-09T06:28:07.752431+0000 mon.a (mon.0) 884 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:28:08.132 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[80002]: cluster 2023-10-09T06:28:06.991683+0000 mgr.y (mgr.25723) 696 : cluster 0 pgmap v672: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:08.132 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[80002]: audit 2023-10-09T06:28:07.752220+0000 mon.a (mon.0) 883 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:28:08.133 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:08 smithi125 bash[80002]: cluster 2023-10-09T06:28:07.752431+0000 mon.a (mon.0) 884 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:28:08.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:08.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:08.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:08 smithi155 bash[59495]: cluster 2023-10-09T06:28:06.991683+0000 mgr.y (mgr.25723) 696 : cluster 0 pgmap v672: 73 pgs: 73 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:28:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:08 smithi155 bash[59495]: audit 2023-10-09T06:28:07.752220+0000 mon.a (mon.0) 883 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:28:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:08 smithi155 bash[59495]: cluster 2023-10-09T06:28:07.752431+0000 mon.a (mon.0) 884 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:28:08.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:08.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:08.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:08.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:08.808 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:28:08.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:08.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:08.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:08.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:08.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:08.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:09.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:09.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:09.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:09.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:09.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:09.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: audit 2023-10-09T06:28:08.063397+0000 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.063503+0000 mon.a (mon.0) 886 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.065779+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.069035+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.069091+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.069150+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.069298+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: audit 2023-10-09T06:28:08.325838+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.125:0/2461253397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: audit 2023-10-09T06:28:08.326360+0000 mon.a (mon.0) 887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.782587+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.782649+0000 mon.a (mon.0) 889 : cluster 1 Cluster is now healthy 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: audit 2023-10-09T06:28:08.792619+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.792805+0000 mon.a (mon.0) 891 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: audit 2023-10-09T06:28:08.063397+0000 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.063503+0000 mon.a (mon.0) 886 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.065779+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.069035+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.069091+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.069150+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.069298+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: audit 2023-10-09T06:28:08.325838+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.125:0/2461253397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: audit 2023-10-09T06:28:08.326360+0000 mon.a (mon.0) 887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.782587+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.782649+0000 mon.a (mon.0) 889 : cluster 1 Cluster is now healthy 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: audit 2023-10-09T06:28:08.792619+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:09 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.792805+0000 mon.a (mon.0) 891 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:28:09.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: audit 2023-10-09T06:28:08.063397+0000 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.125:0/1272706437' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.063503+0000 mon.a (mon.0) 886 : cluster 0 osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.065779+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.069035+0000 osd.4 (osd.4) 86 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.069091+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.069150+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.069298+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e160 with expected crc 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: audit 2023-10-09T06:28:08.325838+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.125:0/2461253397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: audit 2023-10-09T06:28:08.326360+0000 mon.a (mon.0) 887 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.782587+0000 mon.a (mon.0) 888 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.782649+0000 mon.a (mon.0) 889 : cluster 1 Cluster is now healthy 2023-10-09T06:28:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: audit 2023-10-09T06:28:08.792619+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:09.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:09 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.792805+0000 mon.a (mon.0) 891 : cluster 0 osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:28:09.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:09.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:09.531 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:09.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:09.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:09.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:09.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:09.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:09.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:09.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:09.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:10.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:10.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:10.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.795584+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.150 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.796365+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.797284+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.797968+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.798746+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.799019+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.800698+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.800992+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.803316+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.804017+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.804231+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:08.992316+0000 mgr.y (mgr.25723) 697 : cluster 0 pgmap v675: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:10.151 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.798680+0000 mon.a (mon.0) 892 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:10.152 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:10 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.811216+0000 mon.a (mon.0) 893 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:28:10.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:10.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:10.279 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.795584+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.279 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.796365+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.279 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.797284+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.797968+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.798746+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.799019+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.800698+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.800992+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.803316+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.804017+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.804231+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:08.992316+0000 mgr.y (mgr.25723) 697 : cluster 0 pgmap v675: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.798680+0000 mon.a (mon.0) 892 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:10.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.811216+0000 mon.a (mon.0) 893 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.795584+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.796365+0000 osd.4 (osd.4) 87 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.797284+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.797968+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.798746+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.799019+0000 osd.4 (osd.4) 88 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.800698+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.800992+0000 osd.4 (osd.4) 89 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.803316+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.804017+0000 osd.4 (osd.4) 90 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.804231+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e161 with expected crc 2023-10-09T06:28:10.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:08.992316+0000 mgr.y (mgr.25723) 697 : cluster 0 pgmap v675: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:10.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.798680+0000 mon.a (mon.0) 892 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:10.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:10 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.811216+0000 mon.a (mon.0) 893 : cluster 0 osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:28:10.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:10.397 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:10.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:10.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:10.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:10.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:10.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:10.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:10.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:10.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:10.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:10.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:11.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:11.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:11.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:11.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:11.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:11.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:11.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:11 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.812933+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:11 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.816272+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:11 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.819420+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:11 smithi155 bash[59495]: cluster 2023-10-09T06:28:09.887875+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:11 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.816663+0000 mon.a (mon.0) 894 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:28:11.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:11.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:11.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:11] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.812933+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.816272+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.819420+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[77299]: cluster 2023-10-09T06:28:09.887875+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.816663+0000 mon.a (mon.0) 894 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.812933+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.816272+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.819420+0000 osd.4 (osd.4) 91 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[80002]: cluster 2023-10-09T06:28:09.887875+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e162 with expected crc 2023-10-09T06:28:11.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:11 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.816663+0000 mon.a (mon.0) 894 : cluster 0 osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:28:11.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:11.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:11.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:11.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:11.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:11.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:11.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:11.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:11.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:11.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:11.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:28:11.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:12.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:12.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:12.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:12.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:12.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:12.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:12.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.814479+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.818944+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.821542+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.822855+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.823000+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:10.993035+0000 mgr.y (mgr.25723) 698 : cluster 0 pgmap v678: 73 pgs: 2 remapped+peering, 1 peering, 70 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[77299]: cluster 2023-10-09T06:28:11.816119+0000 mon.a (mon.0) 895 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.814479+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.818944+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.821542+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.822855+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.823000+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:10.993035+0000 mgr.y (mgr.25723) 698 : cluster 0 pgmap v678: 73 pgs: 2 remapped+peering, 1 peering, 70 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T06:28:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:12 smithi125 bash[80002]: cluster 2023-10-09T06:28:11.816119+0000 mon.a (mon.0) 895 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.814479+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.818944+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.821542+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.822855+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.823000+0000 osd.4 (osd.4) 92 : cluster 3 failed to encode map e163 with expected crc 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:10.993035+0000 mgr.y (mgr.25723) 698 : cluster 0 pgmap v678: 73 pgs: 2 remapped+peering, 1 peering, 70 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T06:28:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:12 smithi155 bash[59495]: cluster 2023-10-09T06:28:11.816119+0000 mon.a (mon.0) 895 : cluster 0 osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:28:12.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:12.429 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:12.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:12.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:12.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:12.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:12.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:12.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:12.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:12.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:12.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:12.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:13.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:13.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:13.184 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:13.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:13 smithi155 bash[59495]: cluster 2023-10-09T06:28:11.817479+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:13 smithi155 bash[59495]: cluster 2023-10-09T06:28:11.821188+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:13 smithi155 bash[59495]: cluster 2023-10-09T06:28:11.823221+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.193 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:13 smithi155 bash[59495]: cluster 2023-10-09T06:28:11.827511+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:13.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:13.332 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[77299]: cluster 2023-10-09T06:28:11.817479+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.332 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[77299]: cluster 2023-10-09T06:28:11.821188+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.332 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[77299]: cluster 2023-10-09T06:28:11.823221+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[77299]: cluster 2023-10-09T06:28:11.827511+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[80002]: cluster 2023-10-09T06:28:11.817479+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[80002]: cluster 2023-10-09T06:28:11.821188+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[80002]: cluster 2023-10-09T06:28:11.823221+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.333 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:13 smithi125 bash[80002]: cluster 2023-10-09T06:28:11.827511+0000 osd.4 (osd.4) 93 : cluster 3 failed to encode map e164 with expected crc 2023-10-09T06:28:13.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:13.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:13.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76655 5.1 5:8c23abc2:::benchmark_data_smithi125_106927_object76654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.773881+0000", 2023-10-09T06:28:13.523 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 554.74527014199998, 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3256148940000001, 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76655 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.773881+0000", 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.773881+0000", 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.773879+0000", 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100955 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.773980+0000", 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6905000000000002e-05 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.773997+0000", 2023-10-09T06:28:13.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2030000000000003e-06 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.774004+0000", 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0995999999999997e-05 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.774065+0000", 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000165694 2023-10-09T06:28:13.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.774231+0000", 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000196171 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.774427+0000", 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072992049999999996 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.781726+0000", 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013853019999999999 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.783111+0000", 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019297857000000002 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.802409+0000", 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0824e-05 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.802420+0000", 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0006367709999999 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803057+0000", 2023-10-09T06:28:13.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6634e-05 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803074+0000", 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2962049920000001 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099279+0000", 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000217662 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099496+0000", 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3256148940000001 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76663 5.1 5:846b212d:::benchmark_data_smithi125_106927_object76662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.778082+0000", 2023-10-09T06:28:13.532 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 554.74106996399996, 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3215699710000002, 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76663 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778082+0000", 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778082+0000", 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778080+0000", 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3894999999999998e-05 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778134+0000", 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2312e-05 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778146+0000", 2023-10-09T06:28:13.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3029999999999996e-06 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.778152+0000", 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011308030000000001 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.779283+0000", 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011945100000000001 2023-10-09T06:28:13.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.779403+0000", 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018797099999999999 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.779591+0000", 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023478650000000002 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.781938+0000", 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012073749999999999 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.783146+0000", 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019306305999999999 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.802452+0000", 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7090000000000001e-06 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.802457+0000", 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0006559829999999 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803113+0000", 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.329e-06 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803122+0000", 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2964157270000001 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099538+0000", 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113651 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.540 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099651+0000", 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3215699710000002 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76694 5.1 5:8d7024ad:::benchmark_data_smithi125_106927_object76693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e117)", 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:18:58.851125+0000", 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 554.66802624499996, 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.269770023, 2023-10-09T06:28:13.541 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76694 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851125+0000", 2023-10-09T06:28:13.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851125+0000", 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851123+0000", 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9678000000000004e-05 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851213+0000", 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9845999999999999e-05 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851242+0000", 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3607e-05 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851256+0000", 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7857999999999999e-05 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851334+0000", 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.133e-05 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:58.851345+0000", 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95180080600000005 2023-10-09T06:28:13.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803146+0000", 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7719999999999994e-06 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:18:59.803154+0000", 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2965267009999999 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099681+0000", 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000174938 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.099855+0000", 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024868900000000002 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:28:13.547 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.100104+0000", 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081138310000000002 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.108218+0000", 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025162159999999999 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.110734+0000", 2023-10-09T06:28:13.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072383639999999997 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.117973+0000", 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8853e-05 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.118041+0000", 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002853838 2023-10-09T06:28:13.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.120895+0000", 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.269770023 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:112034 5.1 5:8bda5a7f:::benchmark_data_smithi125_106927_object8372:head [delete] snapc 0=[] ondisk+write+known_if_redirected e123)", 2023-10-09T06:28:13.550 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:50.882676+0000", 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 502.636475469, 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13644025900000001, 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 112034 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882676+0000", 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882676+0000", 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882676+0000", 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1949999999999999e-06 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882679+0000", 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4779999999999999e-06 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882684+0000", 2023-10-09T06:28:13.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8229999999999997e-06 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.882688+0000", 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.085403525999999994 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.968092+0000", 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031795800000000002 2023-10-09T06:28:13.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.968409+0000", 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017831999999999999 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.968588+0000", 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018944526 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.555 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.987532+0000", 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9267999999999997e-05 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:50.987572+0000", 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027333128000000002 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.014905+0000", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000241969 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.015147+0000", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039695800000000003 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:51.019116+0000", 2023-10-09T06:28:13.557 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13644025900000001 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:145103 5.1 5:8e561956:::benchmark_data_smithi125_106927_object41441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:20:42.782465+0000", 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 450.73668609700002, 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.169290459, 2023-10-09T06:28:13.558 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 145103 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782465+0000", 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.559 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782465+0000", 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782463+0000", 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0542e-05 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782473+0000", 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0318000000000001e-05 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782494+0000", 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1439999999999992e-06 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.561 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.782503+0000", 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0089995560000000006 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.791502+0000", 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9733000000000003e-05 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for rw locks", 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.791592+0000", 2023-10-09T06:28:13.562 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.10125864800000001 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.892851+0000", 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011250499999999999 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.892963+0000", 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000192587 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.893156+0000", 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.025790042999999999 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.918946+0000", 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8772000000000002e-05 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.918975+0000", 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.030850447 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.949825+0000", 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5259999999999998e-05 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.949860+0000", 2023-10-09T06:28:13.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018955790000000001 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:20:42.951756+0000", 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.169290459 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.566 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:19918 6.e 6:7048b10d:::benchmark_data_smithi125_130224_object19917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:32.255863+0000", 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 281.26328853899997, 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.146289628, 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 19918 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.567 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255863+0000", 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255863+0000", 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255861+0000", 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4960000000000002e-05 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255916+0000", 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0845e-05 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255927+0000", 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9350000000000002e-06 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255932+0000", 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0946000000000003e-05 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.255983+0000", 2023-10-09T06:28:13.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111204 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.256094+0000", 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015315700000000001 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.256247+0000", 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001292968 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.257540+0000", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00058527500000000001 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.258125+0000", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.143812887 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.401938+0000", 2023-10-09T06:28:13.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000101518 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402040+0000", 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011306099999999999 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402153+0000", 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.146289628 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.573 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:19929 6.e 6:7cf0c9f2:::benchmark_data_smithi125_130224_object19928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:32.261019+0000", 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 281.258132375, 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14149133999999999, 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.574 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 19929 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261019+0000", 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261019+0000", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261017+0000", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5871000000000002e-05 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261063+0000", 2023-10-09T06:28:13.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9245999999999998e-05 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261083+0000", 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7450000000000001e-06 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261091+0000", 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1339000000000003e-05 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261163+0000", 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110443 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261273+0000", 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117167 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.261390+0000", 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012344700000000001 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.262625+0000", 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011110100000000001 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.262736+0000", 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13959529700000001 2023-10-09T06:28:13.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402331+0000", 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7880999999999995e-05 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402409+0000", 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000101425 2023-10-09T06:28:13.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402510+0000", 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14149133999999999 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:19935 6.2 6:47403018:::benchmark_data_smithi125_130224_object19934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:32.265277+0000", 2023-10-09T06:28:13.581 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 281.25387474000001, 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13693238099999999, 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 19935 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265277+0000", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265277+0000", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265275+0000", 2023-10-09T06:28:13.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3921999999999998e-05 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265309+0000", 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5883e-05 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265325+0000", 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1740000000000003e-06 2023-10-09T06:28:13.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265332+0000", 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6504000000000001e-05 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265379+0000", 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011975400000000001 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265498+0000", 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106075 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.265604+0000", 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010446209999999999 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.266649+0000", 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8433000000000003e-05 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.266687+0000", 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13527055299999999 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.401958+0000", 2023-10-09T06:28:13.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013532400000000001 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402093+0000", 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011577400000000001 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:32.402209+0000", 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13693238099999999 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.588 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:33151 6.2 6:4f161c2c:::benchmark_data_smithi125_130224_object33150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:39.628099+0000", 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 273.891052081, 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.156372913, 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.589 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 33151 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628099+0000", 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628099+0000", 2023-10-09T06:28:13.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628098+0000", 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1227999999999999e-05 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628139+0000", 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5831000000000001e-05 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628155+0000", 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1160000000000005e-06 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.628164+0000", 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001216429 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.629380+0000", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000158154 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.629538+0000", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000231227 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.629770+0000", 2023-10-09T06:28:13.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014946200000000001 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.631264+0000", 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034698419999999999 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.634734+0000", 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14963910699999999 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.784373+0000", 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6332999999999997e-05 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.784429+0000", 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2868999999999999e-05 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.784472+0000", 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.156372913 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:33164 6.2 6:42eefcc9:::benchmark_data_smithi125_130224_object33163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:39.635924+0000", 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 273.883227433, 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15156783800000001, 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.596 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 33164 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635924+0000", 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635924+0000", 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635922+0000", 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3322000000000003e-05 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635965+0000", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9443999999999999e-05 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635985+0000", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4649e-05 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.636019+0000", 2023-10-09T06:28:13.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0884000000000002e-05 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.636080+0000", 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011491 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.636195+0000", 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018007400000000001 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.636375+0000", 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016314929999999999 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.638007+0000", 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030701700000000001 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.641077+0000", 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.146323386 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.787400+0000", 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0929e-05 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.787451+0000", 2023-10-09T06:28:13.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0621000000000003e-05 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.787492+0000", 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15156783800000001 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.603 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:37216 6.e 6:7f4de956:::benchmark_data_smithi125_130224_object37215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:42.300812+0000", 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 271.21833970300003, 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16278251199999999, 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 37216 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300812+0000", 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300812+0000", 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300810+0000", 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4868999999999999e-05 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300865+0000", 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1214000000000001e-05 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300886+0000", 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2690000000000002e-06 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300893+0000", 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0991000000000001e-05 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.607 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300934+0000", 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5129000000000003e-05 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.301029+0000", 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015258000000000001 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.301182+0000", 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015635740000000001 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.302746+0000", 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00096870699999999997 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.303714+0000", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15970916399999999 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.463423+0000", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103855 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.463527+0000", 2023-10-09T06:28:13.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6987000000000001e-05 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.463594+0000", 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16278251199999999 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.612 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:37226 6.e 6:7d8314c6:::benchmark_data_smithi125_130224_object37225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:42.305078+0000", 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 271.21407365800002, 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.159000593, 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 37226 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.613 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305078+0000", 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305078+0000", 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999638 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.614 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305041+0000", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1730000000000005e-05 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305123+0000", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3884e-05 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305137+0000", 2023-10-09T06:28:13.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9739999999999999e-06 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305143+0000", 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6235e-05 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305189+0000", 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010385799999999999 2023-10-09T06:28:13.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305293+0000", 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020703 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.305500+0000", 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018308700000000001 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.307331+0000", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004913695 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312245+0000", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15170620700000001 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.463951+0000", 2023-10-09T06:28:13.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3679999999999999e-05 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.464025+0000", 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3782000000000001e-05 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.464078+0000", 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.159000593 2023-10-09T06:28:13.619 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:41648 6.e 6:7734ef82:::benchmark_data_smithi125_130224_object41647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:45.719757+0000", 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 267.79939422500001, 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15231067300000001, 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.620 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41648 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719757+0000", 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719757+0000", 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719755+0000", 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6421999999999995e-05 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.622 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719821+0000", 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1857e-05 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719833+0000", 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3489999999999997e-06 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.719840+0000", 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064250899999999996 2023-10-09T06:28:13.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.726265+0000", 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013940299999999999 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.726404+0000", 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019394200000000001 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.726598+0000", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048110669999999996 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.731409+0000", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014920299999999999 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.731558+0000", 2023-10-09T06:28:13.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14028173099999999 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.871840+0000", 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014814400000000001 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.871988+0000", 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9767999999999999e-05 2023-10-09T06:28:13.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.872068+0000", 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15231067300000001 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:41653 6.e 6:7f44fbff:::benchmark_data_smithi125_130224_object41652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:45.722460+0000", 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 267.796691356, 2023-10-09T06:28:13.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.155882624, 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41653 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722460+0000", 2023-10-09T06:28:13.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722460+0000", 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722459+0000", 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000102754 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.629 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722562+0000", 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4632e-05 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722576+0000", 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1559999999999998e-06 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722583+0000", 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057527439999999997 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.728336+0000", 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000132695 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.728469+0000", 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015470800000000001 2023-10-09T06:28:13.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.728624+0000", 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029396940000000001 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.731563+0000", 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071520500000000001 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.732278+0000", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.145908015 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878186+0000", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1545999999999996e-05 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878258+0000", 2023-10-09T06:28:13.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4763000000000001e-05 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878343+0000", 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.155882624 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.634 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:41654 6.e 6:7dc47a93:::benchmark_data_smithi125_130224_object41653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:45.722594+0000", 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 267.79655762700003, 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.156063386, 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41654 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722594+0000", 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722594+0000", 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722593+0000", 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0458999999999999e-05 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722643+0000", 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6880000000000005e-06 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722652+0000", 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4849999999999992e-06 2023-10-09T06:28:13.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.722660+0000", 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0070520009999999996 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.729712+0000", 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013661200000000001 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.729849+0000", 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000185448 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.730034+0000", 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001533855 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.731568+0000", 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00092924700000000002 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.732497+0000", 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14594981900000001 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878447+0000", 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012488899999999999 2023-10-09T06:28:13.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878572+0000", 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5041000000000001e-05 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:45.878657+0000", 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.156063386 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.641 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117543 6.2 6:4782e959:::benchmark_data_smithi125_130224_object6352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.744633+0000", 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.774518167, 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18544629600000001, 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.642 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117543 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744633+0000", 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744633+0000", 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744631+0000", 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0395e-05 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.644 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744642+0000", 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6675000000000003e-05 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744688+0000", 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.772e-06 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744697+0000", 2023-10-09T06:28:13.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1199999999999997e-05 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744758+0000", 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118087 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744876+0000", 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116296 2023-10-09T06:28:13.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744993+0000", 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003828751 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.748821+0000", 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000452535 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749274+0000", 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18066091300000001 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.929935+0000", 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2300999999999999e-05 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.929997+0000", 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2508000000000004e-05 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930080+0000", 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18544629600000001 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.649 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117551 6.2 6:40fa0706:::benchmark_data_smithi125_130224_object6360:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.749902+0000", 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.769249734, 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.180511846, 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117551 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.650 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749902+0000", 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749902+0000", 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749900+0000", 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3489999999999998e-06 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749910+0000", 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2187e-05 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.652 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749922+0000", 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5409999999999997e-06 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749928+0000", 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4246999999999999e-05 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.749963+0000", 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4183000000000004e-05 2023-10-09T06:28:13.653 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.750047+0000", 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8151000000000004e-05 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.750135+0000", 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037948420000000001 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.654 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.753930+0000", 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087153500000000002 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762645+0000", 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.167574577 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.655 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930220+0000", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107386 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930327+0000", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6432000000000002e-05 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930414+0000", 2023-10-09T06:28:13.656 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.180511846 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117558 6.e 6:70ddcbf1:::benchmark_data_smithi125_130224_object6367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.766714+0000", 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 191.75243789699999, 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.163532756, 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.657 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117558 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766714+0000", 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.658 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766714+0000", 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766712+0000", 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6869999999999994e-06 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.659 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766719+0000", 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3309000000000001e-05 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766733+0000", 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6590000000000001e-06 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.660 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766739+0000", 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7578000000000003e-05 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766797+0000", 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000180655 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766977+0000", 2023-10-09T06:28:13.661 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022347099999999999 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.767201+0000", 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040619150000000001 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.771263+0000", 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011691 2023-10-09T06:28:13.662 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.771380+0000", 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15859377699999999 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.929973+0000", 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018530599999999999 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.663 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930159+0000", 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7541999999999996e-05 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930246+0000", 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.163532756 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:154176 6.2 6:403d0085:::benchmark_data_smithi125_130224_object42985:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:52.134494+0000", 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 141.38465720299999, 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13387758499999999, 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 154176 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.665 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134494+0000", 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134494+0000", 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134493+0000", 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2660000000000001e-06 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134501+0000", 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3210999999999999e-05 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134514+0000", 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.114e-06 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134521+0000", 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3650000000000003e-05 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134575+0000", 2023-10-09T06:28:13.668 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112031 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134687+0000", 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012598399999999999 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.134813+0000", 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0044607529999999996 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.669 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.139274+0000", 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024838859999999998 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.141757+0000", 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12648490400000001 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.670 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.268242+0000", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2990999999999994e-05 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.268305+0000", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6554e-05 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:52.268372+0000", 2023-10-09T06:28:13.671 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13387758499999999 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:201804 6.2 6:4f30a3d3:::benchmark_data_smithi125_130224_object90613:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:27:00.200792+0000", 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 73.318359353999995, 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13354721899999999, 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:13.672 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 201804 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200792+0000", 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.673 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200792+0000", 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200791+0000", 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0930000000000003e-06 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.674 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200799+0000", 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4800000000000001e-05 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200813+0000", 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1539999999999999e-06 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:13.675 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200820+0000", 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4484000000000002e-05 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.200874+0000", 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000147374 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.201021+0000", 2023-10-09T06:28:13.676 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017153200000000001 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.201193+0000", 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0035465549999999998 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.204740+0000", 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015374799999999999 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.677 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.204893+0000", 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.12933721400000001 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.334231+0000", 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0937000000000003e-05 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.678 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.334291+0000", 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7904000000000001e-05 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.334339+0000", 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13354721899999999 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:13.679 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:28:13.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:13.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:13.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:13.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:13.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:13.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:13.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:13.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:14.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:14.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:14.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:14.283 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:14.283 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:14.283 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T06:28:14.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T06:28:14.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.339 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:14 smithi155 bash[59495]: cluster 2023-10-09T06:28:12.994081+0000 mgr.y (mgr.25723) 699 : cluster 0 pgmap v680: 73 pgs: 2 remapped+peering, 71 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.1 KiB/s, 4 objects/s recovering 2023-10-09T06:28:14.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:14.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:14 smithi125 bash[77299]: cluster 2023-10-09T06:28:12.994081+0000 mgr.y (mgr.25723) 699 : cluster 0 pgmap v680: 73 pgs: 2 remapped+peering, 71 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.1 KiB/s, 4 objects/s recovering 2023-10-09T06:28:14.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:14 smithi125 bash[80002]: cluster 2023-10-09T06:28:12.994081+0000 mgr.y (mgr.25723) 699 : cluster 0 pgmap v680: 73 pgs: 2 remapped+peering, 71 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 1.1 KiB/s, 4 objects/s recovering 2023-10-09T06:28:14.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:28:14.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:14.621 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:14.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:14.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:14.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:14.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:14.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:14.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:15.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:15.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:15.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:15.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:15.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:15.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:15 smithi155 bash[59495]: cluster 2023-10-09T06:28:15.081277+0000 mon.a (mon.0) 896 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:15.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:15 smithi155 bash[59495]: cluster 2023-10-09T06:28:15.081337+0000 mon.a (mon.0) 897 : cluster 1 Cluster is now healthy 2023-10-09T06:28:15.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:15 smithi125 bash[77299]: cluster 2023-10-09T06:28:15.081277+0000 mon.a (mon.0) 896 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:15.425 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:15 smithi125 bash[77299]: cluster 2023-10-09T06:28:15.081337+0000 mon.a (mon.0) 897 : cluster 1 Cluster is now healthy 2023-10-09T06:28:15.425 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:15 smithi125 bash[80002]: cluster 2023-10-09T06:28:15.081277+0000 mon.a (mon.0) 896 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:15.425 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:15 smithi125 bash[80002]: cluster 2023-10-09T06:28:15.081337+0000 mon.a (mon.0) 897 : cluster 1 Cluster is now healthy 2023-10-09T06:28:15.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:15.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:15.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:15.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:15.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:15.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:15.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:15.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:15.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:15.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:16.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:16.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:16.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:16.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:16.431 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_2' created 2023-10-09T06:28:16.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:16.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:16 smithi125 bash[77299]: cluster 2023-10-09T06:28:14.994700+0000 mgr.y (mgr.25723) 700 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 97 KiB/s, 4 objects/s recovering 2023-10-09T06:28:16.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:16 smithi125 bash[80002]: cluster 2023-10-09T06:28:14.994700+0000 mgr.y (mgr.25723) 700 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 97 KiB/s, 4 objects/s recovering 2023-10-09T06:28:16.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:16.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:16 smithi155 bash[59495]: cluster 2023-10-09T06:28:14.994700+0000 mgr.y (mgr.25723) 700 : cluster 0 pgmap v681: 73 pgs: 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 97 KiB/s, 4 objects/s recovering 2023-10-09T06:28:16.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:16.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:16.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:16.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:16.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:17.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:17.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:17.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:17.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:17.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:17.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:17.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:17.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: audit 2023-10-09T06:28:15.904414+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.125:0/3034035020' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: audit 2023-10-09T06:28:15.904955+0000 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: audit 2023-10-09T06:28:16.416162+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.416378+0000 mon.a (mon.0) 900 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.417159+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.421702+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.424053+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.424441+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.425154+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.426751+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.427329+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.427632+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.430556+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[80002]: cluster 2023-10-09T06:28:16.995254+0000 mgr.y (mgr.25723) 701 : cluster 0 pgmap v683: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 4 objects/s recovering 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: audit 2023-10-09T06:28:15.904414+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.125:0/3034035020' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: audit 2023-10-09T06:28:15.904955+0000 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: audit 2023-10-09T06:28:16.416162+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.416378+0000 mon.a (mon.0) 900 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.417159+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.421702+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.424053+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.424441+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.425154+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.426751+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.427329+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.427632+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.430556+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:17 smithi125 bash[77299]: cluster 2023-10-09T06:28:16.995254+0000 mgr.y (mgr.25723) 701 : cluster 0 pgmap v683: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 4 objects/s recovering 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: audit 2023-10-09T06:28:15.904414+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.125:0/3034035020' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: audit 2023-10-09T06:28:15.904955+0000 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: audit 2023-10-09T06:28:16.416162+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.416378+0000 mon.a (mon.0) 900 : cluster 0 osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.417159+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.421702+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.424053+0000 osd.4 (osd.4) 94 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.424441+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.425154+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.426751+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.427329+0000 osd.4 (osd.4) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.427632+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.430556+0000 osd.4 (osd.4) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:17 smithi155 bash[59495]: cluster 2023-10-09T06:28:16.995254+0000 mgr.y (mgr.25723) 701 : cluster 0 pgmap v683: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 93 KiB/s, 4 objects/s recovering 2023-10-09T06:28:17.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:17.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:17.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:17.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:17.960 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:17.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:18.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T06:28:18.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:18.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:18.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:18.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:18.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:18.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:18.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:18.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:18.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:18.509 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:28:18.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:18.532 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:28:18.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:28:18.585 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:28:18.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:18.614 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:28:18.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:18 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.423268+0000 mon.a (mon.0) 901 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T06:28:18.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:18 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.428393+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:18 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.430107+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:18 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.430277+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:18.640 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:28:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:18.668 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:28:18.690 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:28:18.693 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.693 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.693 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.693 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.694 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.694 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.694 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.694 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.423268+0000 mon.a (mon.0) 901 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.428393+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.430107+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.430277+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.423268+0000 mon.a (mon.0) 901 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.428393+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.430107+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.718 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:18 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.430277+0000 osd.4 (osd.4) 97 : cluster 3 failed to encode map e166 with expected crc 2023-10-09T06:28:18.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:18.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:18.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:18.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:18.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:18.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:19.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:19.135 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:19.225 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:19.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:19.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:19.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:19.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.631440+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: cluster 2023-10-09T06:28:17.634158+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: cluster 2023-10-09T06:28:18.424165+0000 mon.a (mon.0) 902 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: cluster 2023-10-09T06:28:18.426195+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: audit 2023-10-09T06:28:18.475776+0000 mon.a (mon.0) 903 : audit 0 from='client.? 172.21.15.125:0/4169172001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: audit 2023-10-09T06:28:18.587971+0000 mon.a (mon.0) 904 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: audit 2023-10-09T06:28:18.589193+0000 mon.a (mon.0) 905 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: audit 2023-10-09T06:28:18.960656+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[77299]: cluster 2023-10-09T06:28:18.995959+0000 mgr.y (mgr.25723) 702 : cluster 0 pgmap v686: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.631440+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: cluster 2023-10-09T06:28:17.634158+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.538 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: cluster 2023-10-09T06:28:18.424165+0000 mon.a (mon.0) 902 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: cluster 2023-10-09T06:28:18.426195+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: audit 2023-10-09T06:28:18.475776+0000 mon.a (mon.0) 903 : audit 0 from='client.? 172.21.15.125:0/4169172001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: audit 2023-10-09T06:28:18.587971+0000 mon.a (mon.0) 904 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: audit 2023-10-09T06:28:18.589193+0000 mon.a (mon.0) 905 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: audit 2023-10-09T06:28:18.960656+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:19 smithi125 bash[80002]: cluster 2023-10-09T06:28:18.995959+0000 mgr.y (mgr.25723) 702 : cluster 0 pgmap v686: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-10-09T06:28:19.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.631440+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: cluster 2023-10-09T06:28:17.634158+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e165 with expected crc 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: cluster 2023-10-09T06:28:18.424165+0000 mon.a (mon.0) 902 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: cluster 2023-10-09T06:28:18.426195+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: audit 2023-10-09T06:28:18.475776+0000 mon.a (mon.0) 903 : audit 0 from='client.? 172.21.15.125:0/4169172001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: audit 2023-10-09T06:28:18.587971+0000 mon.a (mon.0) 904 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: audit 2023-10-09T06:28:18.589193+0000 mon.a (mon.0) 905 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: audit 2023-10-09T06:28:18.960656+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:19 smithi155 bash[59495]: cluster 2023-10-09T06:28:18.995959+0000 mgr.y (mgr.25723) 702 : cluster 0 pgmap v686: 89 pgs: 8 creating+peering, 3 unknown, 78 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-10-09T06:28:19.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:19.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:19.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:19.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:19.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:19.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:19.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:20.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:20.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:20.202 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:20.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:20.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:20.360 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:20.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[77299]: cluster 2023-10-09T06:28:19.189158+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[77299]: cluster 2023-10-09T06:28:19.189206+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[77299]: cluster 2023-10-09T06:28:19.189330+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[80002]: cluster 2023-10-09T06:28:19.189158+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[80002]: cluster 2023-10-09T06:28:19.189206+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:20 smithi125 bash[80002]: cluster 2023-10-09T06:28:19.189330+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:20.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:20.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:20.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:20 smithi155 bash[59495]: cluster 2023-10-09T06:28:19.189158+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:20 smithi155 bash[59495]: cluster 2023-10-09T06:28:19.189206+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:20 smithi155 bash[59495]: cluster 2023-10-09T06:28:19.189330+0000 osd.4 (osd.4) 98 : cluster 3 failed to encode map e167 with expected crc 2023-10-09T06:28:20.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:20.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:20.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:20.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:21.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:21.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:21.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:21.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:21.298 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:21.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:21] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T06:28:21.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:21.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:21.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:21.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:21 smithi125 bash[77299]: cluster 2023-10-09T06:28:20.996855+0000 mgr.y (mgr.25723) 703 : cluster 0 pgmap v687: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:21.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:21 smithi125 bash[80002]: cluster 2023-10-09T06:28:20.996855+0000 mgr.y (mgr.25723) 703 : cluster 0 pgmap v687: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:21.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:21 smithi155 bash[59495]: cluster 2023-10-09T06:28:20.996855+0000 mgr.y (mgr.25723) 703 : cluster 0 pgmap v687: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:21.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:21.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:21.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:21.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:21.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:21.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:21.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:21.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:22.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:22.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:22.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:22.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:22.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:22.308 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:22.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:22.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:22.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:22.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:28:22.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:22.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:22.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:22.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:22.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:22.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:22.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:22.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:22.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:22.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:23.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:23.082 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:23.082 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:23.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:23.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:23.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:23.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:23.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:23.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:23.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:23.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:23.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:23.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:23.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:23.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:23.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:23.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:23.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:23.920 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:28:23.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:23.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:23.986 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:24.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:24.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:24.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:24.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:24.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:24.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:24 smithi125 bash[77299]: cluster 2023-10-09T06:28:22.997867+0000 mgr.y (mgr.25723) 704 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:24.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:24 smithi125 bash[80002]: cluster 2023-10-09T06:28:22.997867+0000 mgr.y (mgr.25723) 704 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:24.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:24.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:24 smithi155 bash[59495]: cluster 2023-10-09T06:28:22.997867+0000 mgr.y (mgr.25723) 704 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:24.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:24.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:24.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:24.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:24.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:24.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:24.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:24.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:24.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:24.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:24.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:25.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:25.079 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:28:25.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:25.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:25.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:25.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:25.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:25 smithi125 bash[77299]: audit 2023-10-09T06:28:24.213054+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.125:0/1537215774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.348 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:25 smithi125 bash[77299]: audit 2023-10-09T06:28:24.213468+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:25 smithi125 bash[80002]: audit 2023-10-09T06:28:24.213054+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.125:0/1537215774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.348 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:25 smithi125 bash[80002]: audit 2023-10-09T06:28:24.213468+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.360 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:25 smithi155 bash[59495]: audit 2023-10-09T06:28:24.213054+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.125:0/1537215774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:25 smithi155 bash[59495]: audit 2023-10-09T06:28:24.213468+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:25.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:25.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:25.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:25.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:25.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:25.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:25.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:25.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:25.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:25.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:26.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:26.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:24.998862+0000 mgr.y (mgr.25723) 705 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.055458+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: audit 2023-10-09T06:28:25.065497+0000 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.065570+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.232454+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.234546+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.377495+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.377612+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[77299]: cluster 2023-10-09T06:28:25.377744+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:24.998862+0000 mgr.y (mgr.25723) 705 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.055458+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: audit 2023-10-09T06:28:25.065497+0000 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.065570+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.232454+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.234546+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.377495+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.377612+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:26 smithi125 bash[80002]: cluster 2023-10-09T06:28:25.377744+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:26.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:26.288 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:26.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:24.998862+0000 mgr.y (mgr.25723) 705 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.055458+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: audit 2023-10-09T06:28:25.065497+0000 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.065570+0000 mon.a (mon.0) 910 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-10-09T06:28:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.232454+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.234546+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.377495+0000 osd.4 (osd.4) 99 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.377612+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:26 smithi155 bash[59495]: cluster 2023-10-09T06:28:25.377744+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e168 with expected crc 2023-10-09T06:28:26.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:26.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:26.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:26.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:26.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:26.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:26.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:26.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:26.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:26.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:27.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:27.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:27.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:27.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:27.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:27.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:27.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:27.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:27.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:27.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:27.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:27.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:27.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:27.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:28.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:28.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:28.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:28 smithi155 bash[59495]: cluster 2023-10-09T06:28:26.999412+0000 mgr.y (mgr.25723) 706 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:28.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:28.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:28.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:28.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:28.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:28 smithi125 bash[77299]: cluster 2023-10-09T06:28:26.999412+0000 mgr.y (mgr.25723) 706 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:28.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:28 smithi125 bash[80002]: cluster 2023-10-09T06:28:26.999412+0000 mgr.y (mgr.25723) 706 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:28.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:28.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:28.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:28.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:28.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:28.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:28.697 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:28.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:28.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:28.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:28.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:28.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:28.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:29.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:29.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:29.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:28:29.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:29.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:29.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:29.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:29.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:29.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.867 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:29.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:29.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:29.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:29.997 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[77299]: debug 2023-10-09T06:28:29.809+0000 7ff949fd9700 -1 mon.a@0(leader).osd e168 definitely_dead 0 2023-10-09T06:28:30.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:30.011 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.6. 2023-10-09T06:28:30.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:30.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[77299]: audit 2023-10-09T06:28:28.981093+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:30.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[77299]: cluster 2023-10-09T06:28:29.000132+0000 mgr.y (mgr.25723) 707 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:30.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[77299]: audit 2023-10-09T06:28:29.812465+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:28:30.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[80002]: audit 2023-10-09T06:28:28.981093+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:30.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[80002]: cluster 2023-10-09T06:28:29.000132+0000 mgr.y (mgr.25723) 707 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:30.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:29 smithi125 bash[80002]: audit 2023-10-09T06:28:29.812465+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:28:30.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:30.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:30.321 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:29 smithi155 bash[59495]: audit 2023-10-09T06:28:28.981093+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:30.321 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:29 smithi155 bash[59495]: cluster 2023-10-09T06:28:29.000132+0000 mgr.y (mgr.25723) 707 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:30.321 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:29 smithi155 bash[59495]: audit 2023-10-09T06:28:29.812465+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:28:30.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:30.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:30.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:30.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:30.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:30.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:30.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:30.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:30.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:30.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:30.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:31.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:31.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:31.160 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:31.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:30 smithi125 bash[77299]: cluster 2023-10-09T06:28:29.996428+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:30 smithi125 bash[77299]: audit 2023-10-09T06:28:29.996974+0000 mon.a (mon.0) 913 : audit 1 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[77299]: cluster 2023-10-09T06:28:29.997224+0000 mon.a (mon.0) 914 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[77299]: cluster 2023-10-09T06:28:29.999899+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[77299]: cluster 2023-10-09T06:28:29.999916+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.000184+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.000998+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:29.996428+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: audit 2023-10-09T06:28:29.996974+0000 mon.a (mon.0) 913 : audit 1 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:28:31.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:29.997224+0000 mon.a (mon.0) 914 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:28:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:29.999899+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:29.999916+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.000184+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.000998+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:31.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:29.996428+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: audit 2023-10-09T06:28:29.996974+0000 mon.a (mon.0) 913 : audit 1 from='client.? 172.21.15.125:0/1119634178' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:28:31.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:29.997224+0000 mon.a (mon.0) 914 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:28:31.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:29.999899+0000 osd.4 (osd.4) 100 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:29.999916+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.000184+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:31 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.000998+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e169 with expected crc 2023-10-09T06:28:31.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:31.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:31.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:31.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:31.546 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:31] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T06:28:31.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:31.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:31.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:31.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:31.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:31.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:31.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:31.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:31.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:31.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:32.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:32.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.995058+0000 mon.a (mon.0) 915 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:28:32.066 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.997316+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.999232+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.999324+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:30.999943+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.000187+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.001244+0000 mgr.y (mgr.25723) 708 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.001791+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.002610+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.003138+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.003377+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.004048+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.067 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.004854+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.005214+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.005497+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.007529+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.068 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:32 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.994399+0000 mon.a (mon.0) 916 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:28:32.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:32.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:32.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:32.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:32.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:32.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:32.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.995058+0000 mon.a (mon.0) 915 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.997316+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.999232+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.999324+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:30.999943+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.000187+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.001244+0000 mgr.y (mgr.25723) 708 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.001791+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.002610+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.003138+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.003377+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.004048+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.004854+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.005214+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.005497+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.007529+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.994399+0000 mon.a (mon.0) 916 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.995058+0000 mon.a (mon.0) 915 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.997316+0000 osd.4 (osd.4) 101 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.999232+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.999324+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:30.999943+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.000187+0000 osd.4 (osd.4) 102 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.001244+0000 mgr.y (mgr.25723) 708 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.001791+0000 osd.4 (osd.4) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.002610+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.003138+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.003377+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.004048+0000 osd.4 (osd.4) 104 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.004854+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.005214+0000 osd.4 (osd.4) 105 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.005497+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.007529+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e170 with expected crc 2023-10-09T06:28:32.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:32 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.994399+0000 mon.a (mon.0) 916 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:28:32.417 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:32.417 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:32.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:32.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:32.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:32.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:32.672 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T06:28:32.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:32.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:32.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:32.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:32.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:32.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:33.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:33.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:33.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:33.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:33.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:33 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.996038+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:33 smithi155 bash[59495]: cluster 2023-10-09T06:28:31.999639+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:33 smithi155 bash[59495]: cluster 2023-10-09T06:28:32.003401+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:33 smithi155 bash[59495]: cluster 2023-10-09T06:28:32.003773+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.266 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.996038+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.266 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[80002]: cluster 2023-10-09T06:28:31.999639+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.266 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[80002]: cluster 2023-10-09T06:28:32.003401+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.266 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[80002]: cluster 2023-10-09T06:28:32.003773+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.996038+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[77299]: cluster 2023-10-09T06:28:31.999639+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[77299]: cluster 2023-10-09T06:28:32.003401+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.267 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:33 smithi125 bash[77299]: cluster 2023-10-09T06:28:32.003773+0000 osd.4 (osd.4) 106 : cluster 3 failed to encode map e171 with expected crc 2023-10-09T06:28:33.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:33.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:28:33.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:33.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:33.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:33.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:33.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:33.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:33.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:33.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:33.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:33.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:34.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:34.035 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T06:28:34.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:34.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.002038+0000 mgr.y (mgr.25723) 709 : cluster 0 pgmap v697: 89 pgs: 1 active+undersized+remapped, 1 remapped+peering, 87 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:34.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.015962+0000 mon.a (mon.0) 917 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:28:34.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.016672+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.017363+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.020933+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.023501+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: cluster 2023-10-09T06:28:33.025063+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: audit 2023-10-09T06:28:33.081509+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:28:34.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: audit 2023-10-09T06:28:33.587483+0000 mon.a (mon.0) 919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:34.116 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[77299]: audit 2023-10-09T06:28:33.588880+0000 mon.a (mon.0) 920 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.002038+0000 mgr.y (mgr.25723) 709 : cluster 0 pgmap v697: 89 pgs: 1 active+undersized+remapped, 1 remapped+peering, 87 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.015962+0000 mon.a (mon.0) 917 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.016672+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.017363+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.020933+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.023501+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: cluster 2023-10-09T06:28:33.025063+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: audit 2023-10-09T06:28:33.081509+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:28:34.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: audit 2023-10-09T06:28:33.587483+0000 mon.a (mon.0) 919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:34.118 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:34 smithi125 bash[80002]: audit 2023-10-09T06:28:33.588880+0000 mon.a (mon.0) 920 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:34.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:34.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:34.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:34.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.002038+0000 mgr.y (mgr.25723) 709 : cluster 0 pgmap v697: 89 pgs: 1 active+undersized+remapped, 1 remapped+peering, 87 active+clean; 4.6 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail 2023-10-09T06:28:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.015962+0000 mon.a (mon.0) 917 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:28:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.016672+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.017363+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.020933+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.023501+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: cluster 2023-10-09T06:28:33.025063+0000 osd.4 (osd.4) 107 : cluster 3 failed to encode map e172 with expected crc 2023-10-09T06:28:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: audit 2023-10-09T06:28:33.081509+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:28:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: audit 2023-10-09T06:28:33.587483+0000 mon.a (mon.0) 919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:34 smithi155 bash[59495]: audit 2023-10-09T06:28:33.588880+0000 mon.a (mon.0) 920 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:34.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:34.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:34.543 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:34.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:34.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:34.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:34.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:34.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:34.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:34.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:35.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:35.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: audit 2023-10-09T06:28:34.024246+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:28:35.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.024599+0000 mon.a (mon.0) 922 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.027634+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.029313+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.030058+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.031623+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.032359+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:35 smithi155 bash[59495]: cluster 2023-10-09T06:28:34.292008+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:35.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:35.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:35.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: audit 2023-10-09T06:28:34.024246+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:28:35.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.024599+0000 mon.a (mon.0) 922 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:28:35.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.027634+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.029313+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.030058+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.031623+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.032359+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[77299]: cluster 2023-10-09T06:28:34.292008+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: audit 2023-10-09T06:28:34.024246+0000 mon.a (mon.0) 921 : audit 1 from='client.? 172.21.15.125:0/4076312284' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.024599+0000 mon.a (mon.0) 922 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.027634+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.029313+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.030058+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.031623+0000 osd.4 (osd.4) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.032359+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.196 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:35 smithi125 bash[80002]: cluster 2023-10-09T06:28:34.292008+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e173 with expected crc 2023-10-09T06:28:35.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:35.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:35.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:35.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:35.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:35.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:35.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:35.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:35.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:35.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:36.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:36.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:36.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:36.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:36 smithi155 bash[59495]: cluster 2023-10-09T06:28:35.002964+0000 mgr.y (mgr.25723) 710 : cluster 0 pgmap v700: 89 pgs: 1 remapped+peering, 88 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 145 KiB/s, 2 objects/s recovering 2023-10-09T06:28:36.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:36.293 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:36 smithi125 bash[77299]: cluster 2023-10-09T06:28:35.002964+0000 mgr.y (mgr.25723) 710 : cluster 0 pgmap v700: 89 pgs: 1 remapped+peering, 88 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 145 KiB/s, 2 objects/s recovering 2023-10-09T06:28:36.293 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:36 smithi125 bash[80002]: cluster 2023-10-09T06:28:35.002964+0000 mgr.y (mgr.25723) 710 : cluster 0 pgmap v700: 89 pgs: 1 remapped+peering, 88 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 145 KiB/s, 2 objects/s recovering 2023-10-09T06:28:36.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:36.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:36.508 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.539 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:28:36.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:36.566 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:28:36.593 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:28:36.620 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:28:36.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:36.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:28:36.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:36.673 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:28:36.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:36.702 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:28:36.728 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:28:36.730 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.730 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.730 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.730 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.731 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.731 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.731 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.731 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:28:36.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:36.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:36.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:36.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:37.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:37.117 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T06:28:37.120 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:28:37.120 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:28:37.160 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:28:37.161 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:28:37.161 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_153312 2023-10-09T06:28:37.161 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:28:37.161 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:28:37.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:37.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:37.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:37 smithi125 bash[77299]: audit 2023-10-09T06:28:36.505210+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.125:0/3700190507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:37.319 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:37 smithi125 bash[80002]: audit 2023-10-09T06:28:36.505210+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.125:0/3700190507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:37.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:37.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:37 smithi155 bash[59495]: audit 2023-10-09T06:28:36.505210+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.125:0/3700190507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:37.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:37.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:37.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:37.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:37.674 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:37.674 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:37.674 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:28:37.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:28:37.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:37.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:37.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:37.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:37.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:38.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:38.141 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:38.160 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2758 2742 171.365 171.375 0.00528522 0.00581501 2023-10-09T06:28:38.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:38.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:38 smithi155 bash[59495]: cluster 2023-10-09T06:28:37.003767+0000 mgr.y (mgr.25723) 711 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s, 3 objects/s recovering 2023-10-09T06:28:38.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:38.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:38.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:38 smithi125 bash[77299]: cluster 2023-10-09T06:28:37.003767+0000 mgr.y (mgr.25723) 711 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s, 3 objects/s recovering 2023-10-09T06:28:38.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:38 smithi125 bash[80002]: cluster 2023-10-09T06:28:37.003767+0000 mgr.y (mgr.25723) 711 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 4.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s, 3 objects/s recovering 2023-10-09T06:28:38.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:38.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:38.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:38.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:38.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:38.779 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:38.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:38.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:38.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:39.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:39.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:39.160 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5211 5195 162.331 153.312 0.00453085 0.00614904 2023-10-09T06:28:39.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:39.175 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:39 smithi155 bash[59495]: audit 2023-10-09T06:28:38.540522+0000 mon.a (mon.0) 923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:39.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:39.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:39 smithi125 bash[77299]: audit 2023-10-09T06:28:38.540522+0000 mon.a (mon.0) 923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:39.305 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:39 smithi125 bash[80002]: audit 2023-10-09T06:28:38.540522+0000 mon.a (mon.0) 923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:39.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:39.339 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:39.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:39.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:39.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:39.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:39.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:39.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:39.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:28:39.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:39.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:39.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:40.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:40.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:40.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:40 smithi125 bash[77299]: cluster 2023-10-09T06:28:39.004254+0000 mgr.y (mgr.25723) 712 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 25 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s; 83 KiB/s, 2 objects/s recovering 2023-10-09T06:28:40.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:40 smithi125 bash[80002]: cluster 2023-10-09T06:28:39.004254+0000 mgr.y (mgr.25723) 712 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 25 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s; 83 KiB/s, 2 objects/s recovering 2023-10-09T06:28:40.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:40.162 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 7473 7457 155.258 141.375 0.00491438 0.00642771 2023-10-09T06:28:40.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:40.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:40.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:40 smithi155 bash[59495]: cluster 2023-10-09T06:28:39.004254+0000 mgr.y (mgr.25723) 712 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 25 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 46 op/s; 83 KiB/s, 2 objects/s recovering 2023-10-09T06:28:40.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:40.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:40.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:40.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:40.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:40.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:40.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:40.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:40.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:40.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:40.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:41.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:41.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:41.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:41.162 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 9733 9717 151.754 141.25 0.00368042 0.00658005 2023-10-09T06:28:41.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:41.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:41.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:41] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.33.4" 2023-10-09T06:28:41.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:41.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:41.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:41.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:41.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:41.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:41.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:41.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:41.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:42.055 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:42 smithi155 bash[59495]: cluster 2023-10-09T06:28:41.004775+0000 mgr.y (mgr.25723) 713 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 125 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 240 op/s; 73 KiB/s, 3 objects/s recovering 2023-10-09T06:28:42.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:42.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:42.162 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 11808 11792 147.34 129.688 0.00457882 0.00677697 2023-10-09T06:28:42.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:42.226 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:42 smithi125 bash[77299]: cluster 2023-10-09T06:28:41.004775+0000 mgr.y (mgr.25723) 713 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 125 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 240 op/s; 73 KiB/s, 3 objects/s recovering 2023-10-09T06:28:42.227 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:42 smithi125 bash[80002]: cluster 2023-10-09T06:28:41.004775+0000 mgr.y (mgr.25723) 713 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 125 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 240 op/s; 73 KiB/s, 3 objects/s recovering 2023-10-09T06:28:42.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:42.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:42.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:42.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:42.403 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:42.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:42.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:42.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:42.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:42.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:42.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:42.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:42.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:42.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:42.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:42.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:43.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:43.157 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:43.162 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 13678 13662 142.262 116.875 0.00519398 0.00702097 2023-10-09T06:28:43.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:43.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:43.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:43.355 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:28:43.355 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:28:43.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:28:43.355 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:76749 5.7 5:e4410cf9:::benchmark_data_smithi125_106927_object76748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:01.179249+0000", 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 582.165695605, 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19455086299999999, 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 76749 2023-10-09T06:28:43.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179249+0000", 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179249+0000", 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179248+0000", 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013233099999999999 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179380+0000", 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0919999999999998e-06 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179389+0000", 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4850000000000003e-06 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179393+0000", 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4526000000000001e-05 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179428+0000", 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7394999999999998e-05 2023-10-09T06:28:43.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179525+0000", 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010835100000000001 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.179633+0000", 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042166399999999998 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.183850+0000", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5137999999999997e-05 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.183935+0000", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.189603401 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.373538+0000", 2023-10-09T06:28:43.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000155451 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.373694+0000", 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010631799999999999 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:01.373800+0000", 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19455086299999999 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:86982 5.7 5:e87198b4:::benchmark_data_smithi125_106927_object86981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:14.501166+0000", 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 568.84377876200006, 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.181458549, 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 86982 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501166+0000", 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501166+0000", 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501164+0000", 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5167999999999998e-05 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501219+0000", 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1337999999999997e-05 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501251+0000", 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6000000000000003e-06 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501257+0000", 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9120000000000003e-05 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501317+0000", 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010975 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501426+0000", 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014127399999999999 2023-10-09T06:28:43.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.501568+0000", 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039640220000000002 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.505532+0000", 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0014758480000000001 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.507007+0000", 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17551085899999999 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.682518+0000", 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6196000000000002e-05 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.682574+0000", 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0311e-05 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.682625+0000", 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.181458549 2023-10-09T06:28:43.370 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44693.0:86990 5.7 5:ebf5b9b4:::benchmark_data_smithi125_106927_object86989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:19:14.526216+0000", 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 568.81872940100004, 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.159136744, 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.371 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44693", 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1130795622", 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 86990 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.372 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526216+0000", 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526216+0000", 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.373 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526213+0000", 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5434999999999997e-05 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526309+0000", 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8842999999999999e-05 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.374 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526328+0000", 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3459999999999996e-06 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526335+0000", 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1835000000000001e-05 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526387+0000", 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013576199999999999 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526523+0000", 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013692899999999999 2023-10-09T06:28:43.376 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.526660+0000", 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026113780000000001 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.529271+0000", 2023-10-09T06:28:43.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019089980000000001 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.531180+0000", 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.154074184 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.685254+0000", 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3942e-05 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.685308+0000", 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4271999999999999e-05 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:19:14.685352+0000", 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.159136744 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:33160 6.7 6:e8e2a50d:::benchmark_data_smithi125_130224_object33159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:43.380 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:39.633600+0000", 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 303.711344913, 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15159529099999999, 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 33160 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.381 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633600+0000", 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.382 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633600+0000", 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633599+0000", 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.198e-05 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633651+0000", 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1021e-05 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633662+0000", 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2730000000000002e-06 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633667+0000", 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0875000000000002e-05 2023-10-09T06:28:43.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633728+0000", 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1620999999999996e-05 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633819+0000", 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011984700000000001 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633939+0000", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015722519999999999 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.635511+0000", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033342710000000002 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.638846+0000", 2023-10-09T06:28:43.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14619343300000001 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.785039+0000", 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0574000000000004e-05 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.785130+0000", 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5641000000000004e-05 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.785195+0000", 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15159529099999999 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:37210 6.d 6:b8e3bb07:::benchmark_data_smithi125_130224_object37209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:42.297290+0000", 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 301.04765538700002, 2023-10-09T06:28:43.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16115115499999999, 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 37210 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297290+0000", 2023-10-09T06:28:43.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297290+0000", 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297288+0000", 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3841999999999998e-05 2023-10-09T06:28:43.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297332+0000", 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9681999999999999e-05 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297352+0000", 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8630000000000008e-06 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297360+0000", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2737e-05 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297412+0000", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000132819 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297545+0000", 2023-10-09T06:28:43.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019466199999999999 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.297740+0000", 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022408520000000002 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.299981+0000", 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9043999999999999e-05 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.300000+0000", 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15827080900000001 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.458271+0000", 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7304000000000002e-05 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.458368+0000", 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2894999999999999e-05 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.458441+0000", 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16115115499999999 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:37237 6.d 6:bfa3cf39:::benchmark_data_smithi125_130224_object37236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:42.312821+0000", 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 301.032124356, 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14787099300000001, 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 37237 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.396 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312821+0000", 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312821+0000", 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312819+0000", 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000140889 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312960+0000", 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5733999999999999e-05 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312976+0000", 2023-10-09T06:28:43.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5660000000000001e-06 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.312983+0000", 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3378000000000003e-05 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.313037+0000", 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013613300000000001 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.399 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.313173+0000", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016759300000000001 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.313341+0000", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001814846 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.315155+0000", 2023-10-09T06:28:43.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016999770000000001 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.316855+0000", 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.143642772 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.460498+0000", 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107846 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.460606+0000", 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5684000000000002e-05 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:42.460692+0000", 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14787099300000001 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.402 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:39509 6.7 6:ecb54e18:::benchmark_data_smithi125_130224_object39508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:43.960784+0000", 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 299.38416128799997, 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20655544200000001, 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.403 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39509 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960784+0000", 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960784+0000", 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960782+0000", 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9543000000000002e-05 2023-10-09T06:28:43.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960841+0000", 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4438e-05 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960856+0000", 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1280000000000008e-06 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.960864+0000", 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000938012 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.961802+0000", 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000128035 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.961930+0000", 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000167823 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.962098+0000", 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040602030000000001 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.966158+0000", 2023-10-09T06:28:43.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3854e-05 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.966252+0000", 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.200932685 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.167184+0000", 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9457999999999996e-05 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.167274+0000", 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5219e-05 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.167339+0000", 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20655544200000001 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.410 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:39513 6.7 6:e0d16f4c:::benchmark_data_smithi125_130224_object39512:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:43.962970+0000", 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 299.381975479, 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21338596800000001, 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.411 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39513 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.962970+0000", 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.962970+0000", 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.962969+0000", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2132999999999998e-05 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963011+0000", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3459999999999994e-06 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963019+0000", 2023-10-09T06:28:43.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3519999999999997e-06 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963025+0000", 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00060947400000000004 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963634+0000", 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123583 2023-10-09T06:28:43.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963758+0000", 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014536599999999999 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.963903+0000", 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023843919999999999 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.416 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.966287+0000", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018646209999999999 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968152+0000", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20809192900000001 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.176244+0000", 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1935999999999998e-05 2023-10-09T06:28:43.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.176306+0000", 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9648000000000003e-05 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.176355+0000", 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21338596800000001 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.418 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117538 6.7 6:ee746d7c:::benchmark_data_smithi125_130224_object6347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.736040+0000", 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 221.60890544399999, 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19434135399999999, 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.419 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117538 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736040+0000", 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736040+0000", 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736038+0000", 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3170000000000004e-06 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736046+0000", 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3913999999999999e-05 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736059+0000", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0681e-05 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736070+0000", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6882000000000002e-05 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736127+0000", 2023-10-09T06:28:43.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016103999999999999 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736288+0000", 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013678200000000001 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.736425+0000", 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056280870000000004 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.742053+0000", 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002292737 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.744346+0000", 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.185928912 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930275+0000", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0865000000000001e-05 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930335+0000", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5457000000000002e-05 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.930381+0000", 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19434135399999999 2023-10-09T06:28:43.425 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:127128 6.7 6:e67e1aa5:::benchmark_data_smithi125_130224_object15937:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:14.133949+0000", 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 209.21099619099999, 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.161969533, 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.426 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 127128 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133949+0000", 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133949+0000", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133947+0000", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5169999999999999e-06 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133956+0000", 2023-10-09T06:28:43.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4735e-05 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133970+0000", 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5870000000000004e-06 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.133978+0000", 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6264000000000002e-05 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.134034+0000", 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000129422 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.134164+0000", 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000148567 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:28:43.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.134312+0000", 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013091217 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.147403+0000", 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0840000000000001e-05 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.147454+0000", 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.148297395 2023-10-09T06:28:43.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.295752+0000", 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5542000000000001e-05 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.295847+0000", 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1230999999999996e-05 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:14.295918+0000", 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.161969533 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161450 6.d 6:b1bb3c86:::benchmark_data_smithi125_130224_object50259:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.423585+0000", 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.92135962200001, 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22086029400000001, 2023-10-09T06:28:43.433 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161450 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423585+0000", 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423585+0000", 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423584+0000", 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2580000000000001e-06 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423593+0000", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8542e-05 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423612+0000", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.832e-06 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423618+0000", 2023-10-09T06:28:43.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2863999999999998e-05 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423701+0000", 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142616 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423844+0000", 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022788800000000001 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.424072+0000", 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.027392849 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451465+0000", 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8498999999999995e-05 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451543+0000", 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.192621925 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644165+0000", 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015610099999999999 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644321+0000", 2023-10-09T06:28:43.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012453099999999999 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644446+0000", 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22086029400000001 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161451 6.d 6:ba672944:::benchmark_data_smithi125_130224_object50260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.440 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.423636+0000", 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.92130916799999, 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22576344100000001, 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161451 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423636+0000", 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423636+0000", 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423635+0000", 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.943e-06 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423638+0000", 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1404e-05 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423650+0000", 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9620000000000002e-06 2023-10-09T06:28:43.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.423653+0000", 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0059744899999999998 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.429627+0000", 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000158868 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.444 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.429786+0000", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018177199999999999 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.429968+0000", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021676793999999999 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451645+0000", 2023-10-09T06:28:43.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6160000000000001e-06 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451650+0000", 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.197546995 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649197+0000", 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014533300000000001 2023-10-09T06:28:43.446 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649343+0000", 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6718999999999997e-05 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649399+0000", 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22576344100000001 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.447 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161452 6.a 6:5c4cd135:::benchmark_data_smithi125_130224_object50261:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.425729+0000", 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.91921570400001, 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21849133600000001, 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.448 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161452 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425729+0000", 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425729+0000", 2023-10-09T06:28:43.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425728+0000", 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4379999999999998e-06 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425736+0000", 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2727e-05 2023-10-09T06:28:43.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425748+0000", 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5110000000000003e-06 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425757+0000", 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8410000000000004e-05 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425825+0000", 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000154501 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.425980+0000", 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014889600000000001 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:28:43.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426129+0000", 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015114157 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.441243+0000", 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5785000000000005e-05 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.441339+0000", 2023-10-09T06:28:43.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.202668602 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644007+0000", 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011528599999999999 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644123+0000", 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8123999999999998e-05 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644221+0000", 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21849133600000001 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161455 6.d 6:bba25846:::benchmark_data_smithi125_130224_object50264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.426059+0000", 2023-10-09T06:28:43.455 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.91888555899999, 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22355609200000001, 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161455 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426059+0000", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426059+0000", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426058+0000", 2023-10-09T06:28:43.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1249999999999998e-06 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426064+0000", 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2225000000000001e-05 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426077+0000", 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3440000000000003e-06 2023-10-09T06:28:43.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.426084+0000", 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0097095510000000003 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.435794+0000", 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000170908 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.435964+0000", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022676700000000001 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.436191+0000", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015464079 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451655+0000", 2023-10-09T06:28:43.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7905999999999999e-05 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451693+0000", 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.197749763 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649443+0000", 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111859 2023-10-09T06:28:43.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649555+0000", 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0680000000000002e-05 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.649616+0000", 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22355609200000001 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.462 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161457 6.d 6:b03d037c:::benchmark_data_smithi125_130224_object50266:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.434535+0000", 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.910409762, 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22117100100000001, 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.463 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161457 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434535+0000", 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434535+0000", 2023-10-09T06:28:43.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434534+0000", 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7409999999999992e-06 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434541+0000", 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2709e-05 2023-10-09T06:28:43.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434554+0000", 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8839999999999997e-06 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.434561+0000", 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080063319999999997 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.466 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.442567+0000", 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013301799999999999 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.442700+0000", 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014397699999999999 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.442844+0000", 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0089168749999999995 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451761+0000", 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9249999999999998e-06 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451768+0000", 2023-10-09T06:28:43.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.203830872 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.655599+0000", 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7524e-05 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.655656+0000", 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9796000000000002e-05 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.655706+0000", 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22117100100000001 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161463 6.a 6:5b465e37:::benchmark_data_smithi125_130224_object50272:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.444074+0000", 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.90087084800001, 2023-10-09T06:28:43.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20032045400000001, 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161463 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444074+0000", 2023-10-09T06:28:43.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444074+0000", 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444072+0000", 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.952e-06 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444081+0000", 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3801999999999999e-05 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444095+0000", 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2119999999999997e-06 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444102+0000", 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4568999999999999e-05 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444156+0000", 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4400000000000005e-05 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444241+0000", 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2705000000000001e-05 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.444334+0000", 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056600310000000003 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.449994+0000", 2023-10-09T06:28:43.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2945999999999999e-05 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.450007+0000", 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.194298479 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644305+0000", 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5639000000000001e-05 2023-10-09T06:28:43.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644351+0000", 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3961000000000001e-05 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644395+0000", 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20032045400000001 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.477 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161466 6.d 6:b6e08cee:::benchmark_data_smithi125_130224_object50275:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.446260+0000", 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.89868541800001, 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21207773999999999, 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.478 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161466 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446260+0000", 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446260+0000", 2023-10-09T06:28:43.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446258+0000", 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.844e-06 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446265+0000", 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1992e-05 2023-10-09T06:28:43.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446277+0000", 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5089999999999999e-06 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.446284+0000", 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0055546839999999998 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451838+0000", 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121465 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.451960+0000", 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145162 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.452105+0000", 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0054451580000000003 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.457550+0000", 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013171 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.457682+0000", 2023-10-09T06:28:43.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20055872 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.658241+0000", 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3371000000000001e-05 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.658294+0000", 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3371000000000002e-05 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.658337+0000", 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21207773999999999 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:161477 6.a 6:58cc2002:::benchmark_data_smithi125_130224_object50286:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:02.472615+0000", 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 160.87233010899999, 2023-10-09T06:28:43.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17197104499999999, 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 161477 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472615+0000", 2023-10-09T06:28:43.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472615+0000", 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472612+0000", 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4936999999999999e-05 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472627+0000", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.0933999999999999e-05 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472658+0000", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8002000000000001e-05 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472676+0000", 2023-10-09T06:28:43.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0803000000000006e-05 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472747+0000", 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011876599999999999 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472866+0000", 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122769 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.472988+0000", 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0048177339999999997 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.477806+0000", 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025123000000000002 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.478057+0000", 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.166401717 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644459+0000", 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5997e-05 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.491 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644505+0000", 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0964999999999995e-05 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:02.644586+0000", 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17197104499999999 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:191088 6.d 6:b61b406e:::benchmark_data_smithi125_130224_object79897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:45.055307+0000", 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.28963844800001, 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.134974284, 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191088 2023-10-09T06:28:43.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055307+0000", 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055307+0000", 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:28:43.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055304+0000", 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.152e-05 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055316+0000", 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4318e-05 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055340+0000", 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6750000000000008e-06 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055349+0000", 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7750000000000004e-05 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055447+0000", 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023534599999999999 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055682+0000", 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022806200000000001 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055910+0000", 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0073836739999999998 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.063294+0000", 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033127819999999998 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.066606+0000", 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.123549806 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.190156+0000", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5932000000000005e-05 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.190222+0000", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8733999999999997e-05 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.190281+0000", 2023-10-09T06:28:43.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.134974284 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:191090 6.a 6:5c60e28e:::benchmark_data_smithi125_130224_object79899:head [delete] snapc 0=[] ondisk+write+known_if_redirected e154)", 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:26:45.055447+0000", 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 118.289498421, 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.146567383, 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.500 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191090 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055447+0000", 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055447+0000", 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055445+0000", 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1999999999999997e-06 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055453+0000", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.086e-05 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055473+0000", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2529999999999998e-06 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055481+0000", 2023-10-09T06:28:43.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2976000000000001e-05 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055544+0000", 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000153363 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055697+0000", 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014500299999999999 2023-10-09T06:28:43.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.055842+0000", 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0060606430000000001 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.061903+0000", 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020123369999999999 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.063915+0000", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13803790299999999 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.201953+0000", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2004000000000001e-05 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.201985+0000", 2023-10-09T06:28:43.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9060999999999998e-05 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:26:45.202014+0000", 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.146567383 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:43.507 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:28:43.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:43.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:43.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:43.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:43.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:43.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:43.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:43.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:44.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:44.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:44 smithi155 bash[59495]: cluster 2023-10-09T06:28:43.005407+0000 mgr.y (mgr.25723) 714 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 655 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58 KiB/s, 2 objects/s recovering 2023-10-09T06:28:44.060 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:44 smithi155 bash[59495]: audit 2023-10-09T06:28:44.001424+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:44.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 14970 14954 133.475 80.75 0.00496897 0.00748286 2023-10-09T06:28:44.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:44.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:44.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:44.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:44.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:28:44.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:44 smithi125 bash[77299]: cluster 2023-10-09T06:28:43.005407+0000 mgr.y (mgr.25723) 714 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 655 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58 KiB/s, 2 objects/s recovering 2023-10-09T06:28:44.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:44 smithi125 bash[77299]: audit 2023-10-09T06:28:44.001424+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:44.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:44 smithi125 bash[80002]: cluster 2023-10-09T06:28:43.005407+0000 mgr.y (mgr.25723) 714 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 655 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58 KiB/s, 2 objects/s recovering 2023-10-09T06:28:44.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:44 smithi125 bash[80002]: audit 2023-10-09T06:28:44.001424+0000 mon.a (mon.0) 924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:44.443 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:44.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:44.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:44.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:44.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:44.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:44.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:44.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:44.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:44.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:44.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:44.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:45.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:45.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:45.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 16542 16526 129.072 98.25 0.0126854 0.00774096 2023-10-09T06:28:45.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:45.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:45.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:45.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:45.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:45.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:45.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:45.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:45.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:45 smithi155 bash[59495]: audit 2023-10-09T06:28:44.645854+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:45 smithi155 bash[59495]: audit 2023-10-09T06:28:44.657566+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:45 smithi155 bash[59495]: cluster 2023-10-09T06:28:45.008541+0000 mgr.y (mgr.25723) 715 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 795 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 53 KiB/s, 2 objects/s recovering 2023-10-09T06:28:45.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:45.909 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[77299]: audit 2023-10-09T06:28:44.645854+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.909 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[77299]: audit 2023-10-09T06:28:44.657566+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.909 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[77299]: cluster 2023-10-09T06:28:45.008541+0000 mgr.y (mgr.25723) 715 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 795 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 53 KiB/s, 2 objects/s recovering 2023-10-09T06:28:45.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[80002]: audit 2023-10-09T06:28:44.645854+0000 mon.a (mon.0) 925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[80002]: audit 2023-10-09T06:28:44.657566+0000 mon.a (mon.0) 926 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:45.910 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:45 smithi125 bash[80002]: cluster 2023-10-09T06:28:45.008541+0000 mgr.y (mgr.25723) 715 : cluster 0 pgmap v705: 89 pgs: 89 active+clean; 795 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 53 KiB/s, 2 objects/s recovering 2023-10-09T06:28:45.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:45.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:46.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:46.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:46.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:46.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 18236 18220 126.494 105.875 0.00951405 0.00789858 2023-10-09T06:28:46.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:46.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:46.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:46.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:46.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:46.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:46.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:46.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:46.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:46.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:46.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:46.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:46.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:46.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:46.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:46.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:47.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:47.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:47.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 20033 20017 125.075 112.312 0.0119728 0.00798733 2023-10-09T06:28:47.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:47.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:47.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:47.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:47.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:47.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: audit 2023-10-09T06:28:46.319227+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: audit 2023-10-09T06:28:46.334049+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: audit 2023-10-09T06:28:46.901886+0000 mon.a (mon.0) 929 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:47.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: audit 2023-10-09T06:28:46.902990+0000 mon.a (mon.0) 930 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:47.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: audit 2023-10-09T06:28:46.917877+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[77299]: cluster 2023-10-09T06:28:47.008995+0000 mgr.y (mgr.25723) 716 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 996 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:28:47.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: audit 2023-10-09T06:28:46.319227+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: audit 2023-10-09T06:28:46.334049+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: audit 2023-10-09T06:28:46.901886+0000 mon.a (mon.0) 929 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:47.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: audit 2023-10-09T06:28:46.902990+0000 mon.a (mon.0) 930 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:47.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: audit 2023-10-09T06:28:46.917877+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:47 smithi125 bash[80002]: cluster 2023-10-09T06:28:47.008995+0000 mgr.y (mgr.25723) 716 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 996 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:28:47.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:47.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: audit 2023-10-09T06:28:46.319227+0000 mon.a (mon.0) 927 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: audit 2023-10-09T06:28:46.334049+0000 mon.a (mon.0) 928 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: audit 2023-10-09T06:28:46.901886+0000 mon.a (mon.0) 929 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: audit 2023-10-09T06:28:46.902990+0000 mon.a (mon.0) 930 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: audit 2023-10-09T06:28:46.917877+0000 mon.a (mon.0) 931 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:28:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:47 smithi155 bash[59495]: cluster 2023-10-09T06:28:47.008995+0000 mgr.y (mgr.25723) 716 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 996 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:28:47.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:47.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:47.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:47.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:47.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:48.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:48.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:48.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 21953 21937 124.613 120 0.00580387 0.00801764 2023-10-09T06:28:48.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:48.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:48.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:48.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:48.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:48.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:48.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:48.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:48.905 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:48.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:48.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:48.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:48.938 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:48 smithi125 bash[80002]: audit 2023-10-09T06:28:48.581376+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:49.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:49.064 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:49.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:49.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:48 smithi155 bash[59495]: audit 2023-10-09T06:28:48.581376+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:49.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 23567 23551 122.634 100.875 0.00423132 0.00814754 2023-10-09T06:28:49.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:49.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:49.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:49.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:48 smithi125 bash[77299]: audit 2023-10-09T06:28:48.581376+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:49.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:49.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:49.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:49.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:49.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:49.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:49.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:49.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:49.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:49.976 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:49 smithi155 bash[59495]: cluster 2023-10-09T06:28:49.009623+0000 mgr.y (mgr.25723) 717 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:49.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:49.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:50.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:50.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:50.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:50.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:49 smithi125 bash[77299]: cluster 2023-10-09T06:28:49.009623+0000 mgr.y (mgr.25723) 717 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:50.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:49 smithi125 bash[80002]: cluster 2023-10-09T06:28:49.009623+0000 mgr.y (mgr.25723) 717 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:50.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 25343 25327 121.738 111 0.0203195 0.00820853 2023-10-09T06:28:50.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:50.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:50.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:50.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:50.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:50.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:50.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:50.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:50.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:50.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:50.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:50.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:50.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:51.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:51.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:51.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:51.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:51.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 27158 27142 121.144 113.438 0.00989092 0.00824847 2023-10-09T06:28:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:51.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:51.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:51.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:51.430 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:51.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:51.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:51.539 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:28:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:28:51] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-10-09T06:28:51.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:51.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:51.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:51.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:51.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:51.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:51.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:51.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:51.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:52.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:52.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 28457 28441 118.48 81.1875 0.00652827 0.00843496 2023-10-09T06:28:52.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:52.184 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:52 smithi155 bash[59495]: cluster 2023-10-09T06:28:51.010166+0000 mgr.y (mgr.25723) 718 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:28:52.200 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:52.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:52.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:52.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:52 smithi125 bash[80002]: cluster 2023-10-09T06:28:51.010166+0000 mgr.y (mgr.25723) 718 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:28:52.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:52 smithi125 bash[77299]: cluster 2023-10-09T06:28:51.010166+0000 mgr.y (mgr.25723) 718 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:28:52.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.460 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:52.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:52.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:52.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:52.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:52.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:52.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:52.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:52.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:53.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:53.163 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 30136 30120 117.633 104.938 0.01033 0.00849541 2023-10-09T06:28:53.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:53.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:53.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:53.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:53.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.503 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:53.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:53.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:53.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:53.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:53.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:53.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:53.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:53.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:54.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45170.0:31415 7.d e173/169 7:becc802a:::benchmark_data_smithi125_153312_object31414:head v 173'2067, mlcod=173'2063)", 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:54.074214+0000", 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0038463680000000002, 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038699210000000001, 2023-10-09T06:28:54.079 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074214+0000", 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074214+0000", 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:54.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074214+0000", 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1822e-05 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074256+0000", 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7970000000000006e-06 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074264+0000", 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4850000000000003e-06 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.074269+0000", 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026255390000000001 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:54.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.076894+0000", 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6318000000000001e-05 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.076921+0000", 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027064540000000001 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:31425 7.7 7:ecd5e694:::benchmark_data_smithi125_153312_object31424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.00065663899999999999, 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00072985499999999996, 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 31425 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.084 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077402+0000", 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4017000000000001e-05 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077456+0000", 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8720999999999999e-05 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077475+0000", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6710000000000005e-06 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077483+0000", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4039000000000003e-05 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077527+0000", 2023-10-09T06:28:54.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110596 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077638+0000", 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014253000000000001 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077780+0000", 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00037645300000000001 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:54.088 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:28:54.089 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:54.089 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:28:54.089 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:28:54.089 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T06:28:54.089 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:28:54.105 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:54 smithi155 bash[59495]: cluster 2023-10-09T06:28:53.010791+0000 mgr.y (mgr.25723) 719 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:28:54.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 31433 31417 115.481 81.0625 0.0122249 0.00861086 2023-10-09T06:28:54.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:54.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:54.278 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:54 smithi125 bash[77299]: cluster 2023-10-09T06:28:53.010791+0000 mgr.y (mgr.25723) 719 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:28:54.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:54.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:54.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:54.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:54.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:54 smithi125 bash[80002]: cluster 2023-10-09T06:28:53.010791+0000 mgr.y (mgr.25723) 719 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-10-09T06:28:54.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:54.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:54.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:54.837 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:54.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:54.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:55.073 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:55.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:55.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 32767 32751 113.698 83.375 0.00660653 0.00878828 2023-10-09T06:28:55.204 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:55.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:55.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:55.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:28:55.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:55.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:55.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:55.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:55.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:55.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:55.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:55.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:55.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:55.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:55.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:55.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:55.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:55.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:56.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:56.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:56.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:56.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:56.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 34256 34240 112.611 93.0625 0.00296462 0.00885272 2023-10-09T06:28:56.232 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:56.247 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:56 smithi155 bash[59495]: cluster 2023-10-09T06:28:55.011816+0000 mgr.y (mgr.25723) 720 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:56.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:56.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:56.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:56.402 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:56.461 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:56 smithi125 bash[77299]: cluster 2023-10-09T06:28:55.011816+0000 mgr.y (mgr.25723) 720 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:56.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:56 smithi125 bash[80002]: cluster 2023-10-09T06:28:55.011816+0000 mgr.y (mgr.25723) 720 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:28:56.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:56.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:56.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:56.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:56.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:56.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:56.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:56.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:56.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:57.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:57.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:57.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:57.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:28:57.159047+0000 min lat: 0.00135006 max lat: 0.26168 avg lat: 0.00906948 2023-10-09T06:28:57.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:28:57.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 35284 35268 110.193 64.25 0.0101566 0.00906948 2023-10-09T06:28:57.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:57.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:57.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:57.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:57.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:57.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:57.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:57.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:57.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:57.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:57.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:57.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:57.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:57.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:57.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:58.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:58.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:58.104 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:58 smithi125 bash[77299]: cluster 2023-10-09T06:28:57.012645+0000 mgr.y (mgr.25723) 721 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T06:28:58.106 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:28:58.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:58.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:58.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 36639 36623 108.978 84.6875 0.00400166 0.00916985 2023-10-09T06:28:58.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:58.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:58 smithi155 bash[59495]: cluster 2023-10-09T06:28:57.012645+0000 mgr.y (mgr.25723) 721 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T06:28:58.254 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:58 smithi155 bash[59495]: audit 2023-10-09T06:28:57.743504+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:58.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:58.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:58.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:58 smithi125 bash[77299]: audit 2023-10-09T06:28:57.743504+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:58.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:58 smithi125 bash[80002]: cluster 2023-10-09T06:28:57.012645+0000 mgr.y (mgr.25723) 721 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-09T06:28:58.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:58 smithi125 bash[80002]: audit 2023-10-09T06:28:57.743504+0000 mon.a (mon.0) 933 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:58.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:58.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:58.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:58.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:58.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:58.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:58.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:58.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:58.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:58.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:59.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:59.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:59.107 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: cluster 2023-10-09T06:28:58.077060+0000 mon.a (mon.0) 934 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:59.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:59.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 38040 38024 108.004 87.5625 0.00805497 0.00925267 2023-10-09T06:28:59.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: cluster 2023-10-09T06:28:58.077060+0000 mon.a (mon.0) 934 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:59.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: audit 2023-10-09T06:28:58.091195+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:59.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: cluster 2023-10-09T06:28:58.092036+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: cluster 2023-10-09T06:28:58.095028+0000 mon.a (mon.0) 936 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:28:59.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: cluster 2023-10-09T06:28:58.099701+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:28:59 smithi155 bash[59495]: cluster 2023-10-09T06:28:58.107173+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:59.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:59.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:59.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: audit 2023-10-09T06:28:58.091195+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:59.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: cluster 2023-10-09T06:28:58.092036+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: cluster 2023-10-09T06:28:58.095028+0000 mon.a (mon.0) 936 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: cluster 2023-10-09T06:28:58.099701+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[77299]: cluster 2023-10-09T06:28:58.107173+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: cluster 2023-10-09T06:28:58.077060+0000 mon.a (mon.0) 934 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: audit 2023-10-09T06:28:58.091195+0000 mon.a (mon.0) 935 : audit 1 from='client.? 172.21.15.125:0/1014066695' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: cluster 2023-10-09T06:28:58.092036+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: cluster 2023-10-09T06:28:58.095028+0000 mon.a (mon.0) 936 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: cluster 2023-10-09T06:28:58.099701+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:28:59 smithi125 bash[80002]: cluster 2023-10-09T06:28:58.107173+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e174 with expected crc 2023-10-09T06:28:59.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:59.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:59.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:59.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:59.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:59.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:59.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:59.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:59.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:59.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:28:59.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:00.078 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:00.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:00.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:00.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:00.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 39337 39321 106.833 81.0625 0.00666644 0.00934164 2023-10-09T06:29:00.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.225 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:00.272 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:00 smithi125 bash[77299]: cluster 2023-10-09T06:28:59.013204+0000 mgr.y (mgr.25723) 722 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:29:00.272 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:00 smithi125 bash[80002]: cluster 2023-10-09T06:28:59.013204+0000 mgr.y (mgr.25723) 722 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:29:00.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:00.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:00.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:00.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:00 smithi155 bash[59495]: cluster 2023-10-09T06:28:59.013204+0000 mgr.y (mgr.25723) 722 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:29:00.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:00.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:00.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.626 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:00.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:00.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:00.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:00.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:00.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:01.012 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:01.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:01.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:01.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:01.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 40473 40457 105.34 71 0.0190752 0.00948687 2023-10-09T06:29:01.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:01.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:01.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:01.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:01] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-10-09T06:29:01.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:01.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:01.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:01.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:01.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:01.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:01.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:01.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:01.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:01.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:01.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:02.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:02.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:02.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:02.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 41743 41727 104.301 79.375 0.00727905 0.00957914 2023-10-09T06:29:02.172 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:02 smithi155 bash[59495]: cluster 2023-10-09T06:29:01.013694+0000 mgr.y (mgr.25723) 723 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T06:29:02.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:02.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:02.301 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:02 smithi125 bash[77299]: cluster 2023-10-09T06:29:01.013694+0000 mgr.y (mgr.25723) 723 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T06:29:02.301 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:02 smithi125 bash[80002]: cluster 2023-10-09T06:29:01.013694+0000 mgr.y (mgr.25723) 723 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T06:29:02.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:02.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:02.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:02.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:02.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:02.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:02.629 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:29:02.631 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:29:02.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:02.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:02.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:02.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:02.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:02.862 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:29:02.904 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:29:02.946 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:29:02.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:02.984 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:29:02.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:02.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:03.026 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:29:03.063 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:29:03.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.105 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:29:03.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.122 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:03 smithi125 bash[77299]: audit 2023-10-09T06:29:02.787660+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.125:0/2457683945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:03.122 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:03 smithi125 bash[80002]: audit 2023-10-09T06:29:02.787660+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.125:0/2457683945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:03.146 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:29:03.148 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.149 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:29:03.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 42982 42966 103.267 77.4375 0.0199598 0.00967729 2023-10-09T06:29:03.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:03.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:03.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:03.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:03.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:03 smithi155 bash[59495]: audit 2023-10-09T06:29:02.787660+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.125:0/2457683945' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:03.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:03.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:03.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:03.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:03.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:03.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:03.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:03.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:04.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:04.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:04.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[77299]: audit 2023-10-09T06:29:02.600704+0000 mgr.y (mgr.25723) 724 : audit 0 from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:29:04.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[77299]: cluster 2023-10-09T06:29:03.014309+0000 mgr.y (mgr.25723) 725 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:29:04.157 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[77299]: audit 2023-10-09T06:29:03.581572+0000 mon.a (mon.0) 937 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:04.158 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[80002]: audit 2023-10-09T06:29:02.600704+0000 mgr.y (mgr.25723) 724 : audit 0 from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:29:04.158 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[80002]: cluster 2023-10-09T06:29:03.014309+0000 mgr.y (mgr.25723) 725 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:29:04.158 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:04 smithi125 bash[80002]: audit 2023-10-09T06:29:03.581572+0000 mon.a (mon.0) 937 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:04.164 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 44408 44392 102.743 89.125 0.0210692 0.00972644 2023-10-09T06:29:04.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:04.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:04.291 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:04 smithi155 bash[59495]: audit 2023-10-09T06:29:02.600704+0000 mgr.y (mgr.25723) 724 : audit 0 from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:29:04.291 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:04 smithi155 bash[59495]: cluster 2023-10-09T06:29:03.014309+0000 mgr.y (mgr.25723) 725 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T06:29:04.291 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:04 smithi155 bash[59495]: audit 2023-10-09T06:29:03.581572+0000 mon.a (mon.0) 937 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:04.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:04.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:29:04.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:04.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:04.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:04.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:04.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:04.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:04.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:04.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:04.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:04.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:05.014 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:05.015 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:05.015 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":714,"stamp":"2023-10-09T06:29:01.013391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2370574259,"num_objects":36127,"num_object_clones":0,"num_object_copies":108381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36127,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":37205,"num_write_kb":2325495,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":37269,"ondisk_log_size":37269,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9394356,"kb_used_data":6959412,"kb_used_omap":30,"kb_used_meta":2434849,"kb_avail":740566860,"statfs":{"total":767960285184,"available":758340464640,"internally_reserved":0,"allocated":7126437888,"data_stored":7121321962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":2493285487},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,12,5,11,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":984154112,"num_objects":15017,"num_object_clones":0,"num_object_copies":45051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15017,"num_write_kb":961088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003588"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.037907+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:28:34.037907+0000","last_peered":"2023-10-09T06:28:34.037907+0000","last_clean":"2023-10-09T06:28:34.037907+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:28:34.037907+0000","last_undegraded":"2023-10-09T06:28:34.037907+0000","last_fullsized":"2023-10-09T06:28:34.037907+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.022303+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:28:33.022303+0000","last_peered":"2023-10-09T06:28:33.022303+0000","last_clean":"2023-10-09T06:28:33.022303+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:28:33.022303+0000","last_undegraded":"2023-10-09T06:28:33.022303+0000","last_fullsized":"2023-10-09T06:28:33.022303+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.017934+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:28:33.017934+0000","last_peered":"2023-10-09T06:28:33.017934+0000","last_clean":"2023-10-09T06:28:33.017934+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:28:33.017934+0000","last_undegraded":"2023-10-09T06:28:33.017934+0000","last_fullsized":"2023-10-09T06:28:33.017934+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024480+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:28:33.024480+0000","last_peered":"2023-10-09T06:28:33.024480+0000","last_clean":"2023-10-09T06:28:33.024480+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:28:33.024480+0000","last_undegraded":"2023-10-09T06:28:33.024480+0000","last_fullsized":"2023-10-09T06:28:33.024480+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":162,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.021416+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:28:33.021416+0000","last_peered":"2023-10-09T06:28:33.021416+0000","last_clean":"2023-10-09T06:28:33.021416+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:28:33.021416+0000","last_undegraded":"2023-10-09T06:28:33.021416+0000","last_fullsized":"2023-10-09T06:28:33.021416+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.037084+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:28:34.037084+0000","last_peered":"2023-10-09T06:28:34.037084+0000","last_clean":"2023-10-09T06:28:34.037084+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:28:34.037084+0000","last_undegraded":"2023-10-09T06:28:34.037084+0000","last_fullsized":"2023-10-09T06:28:34.037084+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":126,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024805+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:28:33.024805+0000","last_peered":"2023-10-09T06:28:33.024805+0000","last_clean":"2023-10-09T06:28:33.024805+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:28:33.024805+0000","last_undegraded":"2023-10-09T06:28:33.024805+0000","last_fullsized":"2023-10-09T06:28:33.024805+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":178,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.022313+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:28:34.022313+0000","last_peered":"2023-10-09T06:28:34.022313+0000","last_clean":"2023-10-09T06:28:34.022313+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:28:34.022313+0000","last_undegraded":"2023-10-09T06:28:34.022313+0000","last_fullsized":"2023-10-09T06:28:34.022313+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":157,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297652+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:28:34.297652+0000","last_peered":"2023-10-09T06:28:34.297652+0000","last_clean":"2023-10-09T06:28:34.297652+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:28:34.297652+0000","last_undegraded":"2023-10-09T06:28:34.297652+0000","last_fullsized":"2023-10-09T06:28:34.297652+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.022954+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:28:34.022954+0000","last_peered":"2023-10-09T06:28:34.022954+0000","last_clean":"2023-10-09T06:28:34.022954+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:28:34.022954+0000","last_undegraded":"2023-10-09T06:28:34.022954+0000","last_fullsized":"2023-10-09T06:28:34.022954+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.033610+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:28:34.033610+0000","last_peered":"2023-10-09T06:28:34.033610+0000","last_clean":"2023-10-09T06:28:34.033610+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:28:34.033610+0000","last_undegraded":"2023-10-09T06:28:34.033610+0000","last_fullsized":"2023-10-09T06:28:34.033610+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.019210+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:28:33.019210+0000","last_peered":"2023-10-09T06:28:33.019210+0000","last_clean":"2023-10-09T06:28:33.019210+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:28:33.019210+0000","last_undegraded":"2023-10-09T06:28:33.019210+0000","last_fullsized":"2023-10-09T06:28:33.019210+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":222,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038674+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:28:34.038674+0000","last_peered":"2023-10-09T06:28:34.038674+0000","last_clean":"2023-10-09T06:28:34.038674+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:28:34.038674+0000","last_undegraded":"2023-10-09T06:28:34.038674+0000","last_fullsized":"2023-10-09T06:28:34.038674+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038203+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:28:34.038203+0000","last_peered":"2023-10-09T06:28:34.038203+0000","last_clean":"2023-10-09T06:28:34.038203+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:28:34.038203+0000","last_undegraded":"2023-10-09T06:28:34.038203+0000","last_fullsized":"2023-10-09T06:28:34.038203+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":72,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.032960+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:28:34.032960+0000","last_peered":"2023-10-09T06:28:34.032960+0000","last_clean":"2023-10-09T06:28:34.032960+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:28:34.032960+0000","last_undegraded":"2023-10-09T06:28:34.032960+0000","last_fullsized":"2023-10-09T06:28:34.032960+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":165,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.022986+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:28:34.022986+0000","last_peered":"2023-10-09T06:28:34.022986+0000","last_clean":"2023-10-09T06:28:34.022986+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:28:34.022986+0000","last_undegraded":"2023-10-09T06:28:34.022986+0000","last_fullsized":"2023-10-09T06:28:34.022986+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":144,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024169+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:28:33.024169+0000","last_peered":"2023-10-09T06:28:33.024169+0000","last_clean":"2023-10-09T06:28:33.024169+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:28:33.024169+0000","last_undegraded":"2023-10-09T06:28:33.024169+0000","last_fullsized":"2023-10-09T06:28:33.024169+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297504+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:28:34.297504+0000","last_peered":"2023-10-09T06:28:34.297504+0000","last_clean":"2023-10-09T06:28:34.297504+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:28:34.297504+0000","last_undegraded":"2023-10-09T06:28:34.297504+0000","last_fullsized":"2023-10-09T06:28:34.297504+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.032388+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:28:34.032388+0000","last_peered":"2023-10-09T06:28:34.032388+0000","last_clean":"2023-10-09T06:28:34.032388+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:28:34.032388+0000","last_undegraded":"2023-10-09T06:28:34.032388+0000","last_fullsized":"2023-10-09T06:28:34.032388+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":144,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024395+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:28:33.024395+0000","last_peered":"2023-10-09T06:28:33.024395+0000","last_clean":"2023-10-09T06:28:33.024395+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:28:33.024395+0000","last_undegraded":"2023-10-09T06:28:33.024395+0000","last_fullsized":"2023-10-09T06:28:33.024395+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":24,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.036452+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:28:34.036452+0000","last_peered":"2023-10-09T06:28:34.036452+0000","last_clean":"2023-10-09T06:28:34.036452+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:28:34.036452+0000","last_undegraded":"2023-10-09T06:28:34.036452+0000","last_fullsized":"2023-10-09T06:28:34.036452+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":177,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.022217+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:28:33.022217+0000","last_peered":"2023-10-09T06:28:33.022217+0000","last_clean":"2023-10-09T06:28:33.022217+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:28:33.022217+0000","last_undegraded":"2023-10-09T06:28:33.022217+0000","last_fullsized":"2023-10-09T06:28:33.022217+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"173'2231","reported_seq":2257,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.970834+0000","last_change":"2023-10-09T06:28:31.009442+0000","last_active":"2023-10-09T06:28:56.970834+0000","last_peered":"2023-10-09T06:28:56.970834+0000","last_clean":"2023-10-09T06:28:56.970834+0000","last_became_active":"2023-10-09T06:28:31.009110+0000","last_became_peered":"2023-10-09T06:28:31.009110+0000","last_unstale":"2023-10-09T06:28:56.970834+0000","last_undegraded":"2023-10-09T06:28:56.970834+0000","last_fullsized":"2023-10-09T06:28:56.970834+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2231,"log_dups_size":0,"ondisk_log_size":2231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:27.545767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146210816,"num_objects":2231,"num_object_clones":0,"num_object_copies":6693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2231,"num_write_kb":142784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.033847+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:28:34.033847+0000","last_peered":"2023-10-09T06:28:34.033847+0000","last_clean":"2023-10-09T06:28:34.033847+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:28:34.033847+0000","last_undegraded":"2023-10-09T06:28:34.033847+0000","last_fullsized":"2023-10-09T06:28:34.033847+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.296913+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:28:34.296913+0000","last_peered":"2023-10-09T06:28:34.296913+0000","last_clean":"2023-10-09T06:28:34.296913+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:28:34.296913+0000","last_undegraded":"2023-10-09T06:28:34.296913+0000","last_fullsized":"2023-10-09T06:28:34.296913+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"173'2213","reported_seq":2230,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.772985+0000","last_change":"2023-10-09T06:28:17.434903+0000","last_active":"2023-10-09T06:28:56.772985+0000","last_peered":"2023-10-09T06:28:56.772985+0000","last_clean":"2023-10-09T06:28:56.772985+0000","last_became_active":"2023-10-09T06:28:17.434683+0000","last_became_peered":"2023-10-09T06:28:17.434683+0000","last_unstale":"2023-10-09T06:28:56.772985+0000","last_undegraded":"2023-10-09T06:28:56.772985+0000","last_fullsized":"2023-10-09T06:28:56.772985+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2213,"log_dups_size":0,"ondisk_log_size":2213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:28.684931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2213,"num_write_kb":141632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":70,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.033474+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:28:34.033474+0000","last_peered":"2023-10-09T06:28:34.033474+0000","last_clean":"2023-10-09T06:28:34.033474+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:28:34.033474+0000","last_undegraded":"2023-10-09T06:28:34.033474+0000","last_fullsized":"2023-10-09T06:28:34.033474+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038124+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:28:34.038124+0000","last_peered":"2023-10-09T06:28:34.038124+0000","last_clean":"2023-10-09T06:28:34.038124+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:28:34.038124+0000","last_undegraded":"2023-10-09T06:28:34.038124+0000","last_fullsized":"2023-10-09T06:28:34.038124+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"173'2279","reported_seq":2296,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.642665+0000","last_change":"2023-10-09T06:28:17.441718+0000","last_active":"2023-10-09T06:28:57.642665+0000","last_peered":"2023-10-09T06:28:57.642665+0000","last_clean":"2023-10-09T06:28:57.642665+0000","last_became_active":"2023-10-09T06:28:17.441394+0000","last_became_peered":"2023-10-09T06:28:17.441394+0000","last_unstale":"2023-10-09T06:28:57.642665+0000","last_undegraded":"2023-10-09T06:28:57.642665+0000","last_fullsized":"2023-10-09T06:28:57.642665+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2279,"log_dups_size":0,"ondisk_log_size":2279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:26.409909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149356544,"num_objects":2279,"num_object_clones":0,"num_object_copies":6837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2279,"num_write_kb":145856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":96,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.033812+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:28:34.033812+0000","last_peered":"2023-10-09T06:28:34.033812+0000","last_clean":"2023-10-09T06:28:34.033812+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:28:34.033812+0000","last_undegraded":"2023-10-09T06:28:34.033812+0000","last_fullsized":"2023-10-09T06:28:34.033812+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.036261+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:28:34.036261+0000","last_peered":"2023-10-09T06:28:34.036261+0000","last_clean":"2023-10-09T06:28:34.036261+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:28:34.036261+0000","last_undegraded":"2023-10-09T06:28:34.036261+0000","last_fullsized":"2023-10-09T06:28:34.036261+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"173'2230","reported_seq":2247,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.033124+0000","last_change":"2023-10-09T06:28:17.440958+0000","last_active":"2023-10-09T06:28:57.033124+0000","last_peered":"2023-10-09T06:28:57.033124+0000","last_clean":"2023-10-09T06:28:57.033124+0000","last_became_active":"2023-10-09T06:28:17.440578+0000","last_became_peered":"2023-10-09T06:28:17.440578+0000","last_unstale":"2023-10-09T06:28:57.033124+0000","last_undegraded":"2023-10-09T06:28:57.033124+0000","last_fullsized":"2023-10-09T06:28:57.033124+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2230,"log_dups_size":0,"ondisk_log_size":2230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.677500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146145280,"num_objects":2230,"num_object_clones":0,"num_object_copies":6690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2230,"num_write_kb":142720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":157,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297401+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:28:34.297401+0000","last_peered":"2023-10-09T06:28:34.297401+0000","last_clean":"2023-10-09T06:28:34.297401+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:28:34.297401+0000","last_undegraded":"2023-10-09T06:28:34.297401+0000","last_fullsized":"2023-10-09T06:28:34.297401+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.029454+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:28:34.029454+0000","last_peered":"2023-10-09T06:28:34.029454+0000","last_clean":"2023-10-09T06:28:34.029454+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:28:34.029454+0000","last_undegraded":"2023-10-09T06:28:34.029454+0000","last_fullsized":"2023-10-09T06:28:34.029454+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"173'2248","reported_seq":2265,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.592955+0000","last_change":"2023-10-09T06:28:17.434701+0000","last_active":"2023-10-09T06:28:57.592955+0000","last_peered":"2023-10-09T06:28:57.592955+0000","last_clean":"2023-10-09T06:28:57.592955+0000","last_became_active":"2023-10-09T06:28:17.434374+0000","last_became_peered":"2023-10-09T06:28:17.434374+0000","last_unstale":"2023-10-09T06:28:57.592955+0000","last_undegraded":"2023-10-09T06:28:57.592955+0000","last_fullsized":"2023-10-09T06:28:57.592955+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2248,"log_dups_size":0,"ondisk_log_size":2248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:08.826742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2248,"num_write_kb":143872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":146,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.021895+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:28:33.021895+0000","last_peered":"2023-10-09T06:28:33.021895+0000","last_clean":"2023-10-09T06:28:33.021895+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:28:33.021895+0000","last_undegraded":"2023-10-09T06:28:33.021895+0000","last_fullsized":"2023-10-09T06:28:33.021895+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038593+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:28:34.038593+0000","last_peered":"2023-10-09T06:28:34.038593+0000","last_clean":"2023-10-09T06:28:34.038593+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:28:34.038593+0000","last_undegraded":"2023-10-09T06:28:34.038593+0000","last_fullsized":"2023-10-09T06:28:34.038593+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"173'2159","reported_seq":2176,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.027571+0000","last_change":"2023-10-09T06:28:17.433705+0000","last_active":"2023-10-09T06:28:57.027571+0000","last_peered":"2023-10-09T06:28:57.027571+0000","last_clean":"2023-10-09T06:28:57.027571+0000","last_became_active":"2023-10-09T06:28:17.433400+0000","last_became_peered":"2023-10-09T06:28:17.433400+0000","last_unstale":"2023-10-09T06:28:57.027571+0000","last_undegraded":"2023-10-09T06:28:57.027571+0000","last_fullsized":"2023-10-09T06:28:57.027571+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2159,"log_dups_size":0,"ondisk_log_size":2159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:15.103678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492224,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2159,"num_write_kb":138176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038025+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:28:34.038025+0000","last_peered":"2023-10-09T06:28:34.038025+0000","last_clean":"2023-10-09T06:28:34.038025+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:28:34.038025+0000","last_undegraded":"2023-10-09T06:28:34.038025+0000","last_fullsized":"2023-10-09T06:28:34.038025+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038227+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:28:34.038227+0000","last_peered":"2023-10-09T06:28:34.038227+0000","last_clean":"2023-10-09T06:28:34.038227+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:28:34.038227+0000","last_undegraded":"2023-10-09T06:28:34.038227+0000","last_fullsized":"2023-10-09T06:28:34.038227+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"173'2295","reported_seq":2321,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.945563+0000","last_change":"2023-10-09T06:28:31.008498+0000","last_active":"2023-10-09T06:28:56.945563+0000","last_peered":"2023-10-09T06:28:56.945563+0000","last_clean":"2023-10-09T06:28:56.945563+0000","last_became_active":"2023-10-09T06:28:31.008138+0000","last_became_peered":"2023-10-09T06:28:31.008138+0000","last_unstale":"2023-10-09T06:28:56.945563+0000","last_undegraded":"2023-10-09T06:28:56.945563+0000","last_fullsized":"2023-10-09T06:28:56.945563+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2295,"log_dups_size":0,"ondisk_log_size":2295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:37.724845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150405120,"num_objects":2295,"num_object_clones":0,"num_object_copies":6885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2295,"num_write_kb":146880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.037807+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:28:34.037807+0000","last_peered":"2023-10-09T06:28:34.037807+0000","last_clean":"2023-10-09T06:28:34.037807+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:28:34.037807+0000","last_undegraded":"2023-10-09T06:28:34.037807+0000","last_fullsized":"2023-10-09T06:28:34.037807+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.039068+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:28:34.039068+0000","last_peered":"2023-10-09T06:28:34.039068+0000","last_clean":"2023-10-09T06:28:34.039068+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:28:34.039068+0000","last_undegraded":"2023-10-09T06:28:34.039068+0000","last_fullsized":"2023-10-09T06:28:34.039068+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"173'2178","reported_seq":2202,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.755442+0000","last_change":"2023-10-09T06:28:31.012676+0000","last_active":"2023-10-09T06:28:56.755442+0000","last_peered":"2023-10-09T06:28:56.755442+0000","last_clean":"2023-10-09T06:28:56.755442+0000","last_became_active":"2023-10-09T06:28:31.012376+0000","last_became_peered":"2023-10-09T06:28:31.012376+0000","last_unstale":"2023-10-09T06:28:56.755442+0000","last_undegraded":"2023-10-09T06:28:56.755442+0000","last_fullsized":"2023-10-09T06:28:56.755442+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2178,"log_dups_size":0,"ondisk_log_size":2178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:39.025841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142737408,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2178,"num_write_kb":139392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":166,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.021568+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:28:33.021568+0000","last_peered":"2023-10-09T06:28:33.021568+0000","last_clean":"2023-10-09T06:28:33.021568+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:28:33.021568+0000","last_undegraded":"2023-10-09T06:28:33.021568+0000","last_fullsized":"2023-10-09T06:28:33.021568+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.018753+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:28:33.018753+0000","last_peered":"2023-10-09T06:28:33.018753+0000","last_clean":"2023-10-09T06:28:33.018753+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:28:33.018753+0000","last_undegraded":"2023-10-09T06:28:33.018753+0000","last_fullsized":"2023-10-09T06:28:33.018753+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"174'2344","reported_seq":2370,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:28:59.598676+0000","last_change":"2023-10-09T06:28:31.028689+0000","last_active":"2023-10-09T06:28:59.598676+0000","last_peered":"2023-10-09T06:28:59.598676+0000","last_clean":"2023-10-09T06:28:59.598676+0000","last_became_active":"2023-10-09T06:28:31.027894+0000","last_became_peered":"2023-10-09T06:28:31.027894+0000","last_unstale":"2023-10-09T06:28:59.598676+0000","last_undegraded":"2023-10-09T06:28:59.598676+0000","last_fullsized":"2023-10-09T06:28:59.598676+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2344,"log_dups_size":0,"ondisk_log_size":2344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153616384,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2344,"num_write_kb":150016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1024,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.020978+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:28:33.020978+0000","last_peered":"2023-10-09T06:28:33.020978+0000","last_clean":"2023-10-09T06:28:33.020978+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:28:33.020978+0000","last_undegraded":"2023-10-09T06:28:33.020978+0000","last_fullsized":"2023-10-09T06:28:33.020978+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297625+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:28:34.297625+0000","last_peered":"2023-10-09T06:28:34.297625+0000","last_clean":"2023-10-09T06:28:34.297625+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:28:34.297625+0000","last_undegraded":"2023-10-09T06:28:34.297625+0000","last_fullsized":"2023-10-09T06:28:34.297625+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.019039+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:28:33.019039+0000","last_peered":"2023-10-09T06:28:33.019039+0000","last_clean":"2023-10-09T06:28:33.019039+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:28:33.019039+0000","last_undegraded":"2023-10-09T06:28:33.019039+0000","last_fullsized":"2023-10-09T06:28:33.019039+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"174'2416","reported_seq":2427,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:28:59.789113+0000","last_change":"2023-10-09T06:28:31.020101+0000","last_active":"2023-10-09T06:28:59.789113+0000","last_peered":"2023-10-09T06:28:59.789113+0000","last_clean":"2023-10-09T06:28:59.789113+0000","last_became_active":"2023-10-09T06:28:31.019456+0000","last_became_peered":"2023-10-09T06:28:31.019456+0000","last_unstale":"2023-10-09T06:28:59.789113+0000","last_undegraded":"2023-10-09T06:28:59.789113+0000","last_fullsized":"2023-10-09T06:28:59.789113+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2416,"log_dups_size":0,"ondisk_log_size":2416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158334976,"num_objects":2416,"num_object_clones":0,"num_object_copies":7248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2416,"num_write_kb":154624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":812,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297236+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:28:34.297236+0000","last_peered":"2023-10-09T06:28:34.297236+0000","last_clean":"2023-10-09T06:28:34.297236+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:28:34.297236+0000","last_undegraded":"2023-10-09T06:28:34.297236+0000","last_fullsized":"2023-10-09T06:28:34.297236+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":157,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038071+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:28:34.038071+0000","last_peered":"2023-10-09T06:28:34.038071+0000","last_clean":"2023-10-09T06:28:34.038071+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:28:34.038071+0000","last_undegraded":"2023-10-09T06:28:34.038071+0000","last_fullsized":"2023-10-09T06:28:34.038071+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":139,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.020889+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:28:33.020889+0000","last_peered":"2023-10-09T06:28:33.020889+0000","last_clean":"2023-10-09T06:28:33.020889+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:28:33.020889+0000","last_undegraded":"2023-10-09T06:28:33.020889+0000","last_fullsized":"2023-10-09T06:28:33.020889+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"173'2182","reported_seq":2199,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.627043+0000","last_change":"2023-10-09T06:28:17.440797+0000","last_active":"2023-10-09T06:28:57.627043+0000","last_peered":"2023-10-09T06:28:57.627043+0000","last_clean":"2023-10-09T06:28:57.627043+0000","last_became_active":"2023-10-09T06:28:17.440487+0000","last_became_peered":"2023-10-09T06:28:17.440487+0000","last_unstale":"2023-10-09T06:28:57.627043+0000","last_undegraded":"2023-10-09T06:28:57.627043+0000","last_fullsized":"2023-10-09T06:28:57.627043+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2182,"log_dups_size":0,"ondisk_log_size":2182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:07.232771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142999552,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2182,"num_write_kb":139648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":535,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.021912+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:28:34.021912+0000","last_peered":"2023-10-09T06:28:34.021912+0000","last_clean":"2023-10-09T06:28:34.021912+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:28:34.021912+0000","last_undegraded":"2023-10-09T06:28:34.021912+0000","last_fullsized":"2023-10-09T06:28:34.021912+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.017558+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:28:33.017558+0000","last_peered":"2023-10-09T06:28:33.017558+0000","last_clean":"2023-10-09T06:28:33.017558+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:28:33.017558+0000","last_undegraded":"2023-10-09T06:28:33.017558+0000","last_fullsized":"2023-10-09T06:28:33.017558+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.027216+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:28:34.027216+0000","last_peered":"2023-10-09T06:28:34.027216+0000","last_clean":"2023-10-09T06:28:34.027216+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:28:34.027216+0000","last_undegraded":"2023-10-09T06:28:34.027216+0000","last_fullsized":"2023-10-09T06:28:34.027216+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"173'2190","reported_seq":2207,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:57.630389+0000","last_change":"2023-10-09T06:28:17.433836+0000","last_active":"2023-10-09T06:28:57.630389+0000","last_peered":"2023-10-09T06:28:57.630389+0000","last_clean":"2023-10-09T06:28:57.630389+0000","last_became_active":"2023-10-09T06:28:17.433456+0000","last_became_peered":"2023-10-09T06:28:17.433456+0000","last_unstale":"2023-10-09T06:28:57.630389+0000","last_undegraded":"2023-10-09T06:28:57.630389+0000","last_fullsized":"2023-10-09T06:28:57.630389+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2190,"log_dups_size":0,"ondisk_log_size":2190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:51.314504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143523840,"num_objects":2190,"num_object_clones":0,"num_object_copies":6570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2190,"num_write_kb":140160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":522,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297298+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:28:34.297298+0000","last_peered":"2023-10-09T06:28:34.297298+0000","last_clean":"2023-10-09T06:28:34.297298+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:28:34.297298+0000","last_undegraded":"2023-10-09T06:28:34.297298+0000","last_fullsized":"2023-10-09T06:28:34.297298+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":416,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038903+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:28:34.038903+0000","last_peered":"2023-10-09T06:28:34.038903+0000","last_clean":"2023-10-09T06:28:34.038903+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:28:34.038903+0000","last_undegraded":"2023-10-09T06:28:34.038903+0000","last_fullsized":"2023-10-09T06:28:34.038903+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":156,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.021952+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:28:33.021952+0000","last_peered":"2023-10-09T06:28:33.021952+0000","last_clean":"2023-10-09T06:28:33.021952+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:28:33.021952+0000","last_undegraded":"2023-10-09T06:28:33.021952+0000","last_fullsized":"2023-10-09T06:28:33.021952+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038355+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:28:34.038355+0000","last_peered":"2023-10-09T06:28:34.038355+0000","last_clean":"2023-10-09T06:28:34.038355+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:28:34.038355+0000","last_undegraded":"2023-10-09T06:28:34.038355+0000","last_fullsized":"2023-10-09T06:28:34.038355+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"174'2339","reported_seq":2356,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:28:59.625617+0000","last_change":"2023-10-09T06:28:17.443122+0000","last_active":"2023-10-09T06:28:59.625617+0000","last_peered":"2023-10-09T06:28:59.625617+0000","last_clean":"2023-10-09T06:28:59.625617+0000","last_became_active":"2023-10-09T06:28:17.442723+0000","last_became_peered":"2023-10-09T06:28:17.442723+0000","last_unstale":"2023-10-09T06:28:59.625617+0000","last_undegraded":"2023-10-09T06:28:59.625617+0000","last_fullsized":"2023-10-09T06:28:59.625617+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2339,"log_dups_size":0,"ondisk_log_size":2339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153288704,"num_objects":2339,"num_object_clones":0,"num_object_copies":7017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2339,"num_write_kb":149696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":95,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.032303+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:28:34.032303+0000","last_peered":"2023-10-09T06:28:34.032303+0000","last_clean":"2023-10-09T06:28:34.032303+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:28:34.032303+0000","last_undegraded":"2023-10-09T06:28:34.032303+0000","last_fullsized":"2023-10-09T06:28:34.032303+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.023955+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:28:33.023955+0000","last_peered":"2023-10-09T06:28:33.023955+0000","last_clean":"2023-10-09T06:28:33.023955+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:28:33.023955+0000","last_undegraded":"2023-10-09T06:28:33.023955+0000","last_fullsized":"2023-10-09T06:28:33.023955+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":142,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024569+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:28:33.024569+0000","last_peered":"2023-10-09T06:28:33.024569+0000","last_clean":"2023-10-09T06:28:33.024569+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:28:33.024569+0000","last_undegraded":"2023-10-09T06:28:33.024569+0000","last_fullsized":"2023-10-09T06:28:33.024569+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"174'2414","reported_seq":2430,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:28:59.582077+0000","last_change":"2023-10-09T06:28:17.440955+0000","last_active":"2023-10-09T06:28:59.582077+0000","last_peered":"2023-10-09T06:28:59.582077+0000","last_clean":"2023-10-09T06:28:59.582077+0000","last_became_active":"2023-10-09T06:28:17.440657+0000","last_became_peered":"2023-10-09T06:28:17.440657+0000","last_unstale":"2023-10-09T06:28:59.582077+0000","last_undegraded":"2023-10-09T06:28:59.582077+0000","last_fullsized":"2023-10-09T06:28:59.582077+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2414,"log_dups_size":0,"ondisk_log_size":2414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158203904,"num_objects":2414,"num_object_clones":0,"num_object_copies":7242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2414,"num_write_kb":154496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":84,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.033843+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:28:34.033843+0000","last_peered":"2023-10-09T06:28:34.033843+0000","last_clean":"2023-10-09T06:28:34.033843+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:28:34.033843+0000","last_undegraded":"2023-10-09T06:28:34.033843+0000","last_fullsized":"2023-10-09T06:28:34.033843+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.020939+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:28:33.020939+0000","last_peered":"2023-10-09T06:28:33.020939+0000","last_clean":"2023-10-09T06:28:33.020939+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:28:33.020939+0000","last_undegraded":"2023-10-09T06:28:33.020939+0000","last_fullsized":"2023-10-09T06:28:33.020939+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":139,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024281+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:28:33.024281+0000","last_peered":"2023-10-09T06:28:33.024281+0000","last_clean":"2023-10-09T06:28:33.024281+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:28:33.024281+0000","last_undegraded":"2023-10-09T06:28:33.024281+0000","last_fullsized":"2023-10-09T06:28:33.024281+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"173'2207","reported_seq":2224,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.764203+0000","last_change":"2023-10-09T06:28:17.441380+0000","last_active":"2023-10-09T06:28:56.764203+0000","last_peered":"2023-10-09T06:28:56.764203+0000","last_clean":"2023-10-09T06:28:56.764203+0000","last_became_active":"2023-10-09T06:28:17.440129+0000","last_became_peered":"2023-10-09T06:28:17.440129+0000","last_unstale":"2023-10-09T06:28:56.764203+0000","last_undegraded":"2023-10-09T06:28:56.764203+0000","last_fullsized":"2023-10-09T06:28:56.764203+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2207,"log_dups_size":0,"ondisk_log_size":2207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:24:53.884579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2207,"num_write_kb":141248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":172,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.022131+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:28:33.022131+0000","last_peered":"2023-10-09T06:28:33.022131+0000","last_clean":"2023-10-09T06:28:33.022131+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:28:33.022131+0000","last_undegraded":"2023-10-09T06:28:33.022131+0000","last_fullsized":"2023-10-09T06:28:33.022131+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":554,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.035790+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:28:34.035790+0000","last_peered":"2023-10-09T06:28:34.035790+0000","last_clean":"2023-10-09T06:28:34.035790+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:28:34.035790+0000","last_undegraded":"2023-10-09T06:28:34.035790+0000","last_fullsized":"2023-10-09T06:28:34.035790+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":171,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.022167+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:28:34.022167+0000","last_peered":"2023-10-09T06:28:34.022167+0000","last_clean":"2023-10-09T06:28:34.022167+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:28:34.022167+0000","last_undegraded":"2023-10-09T06:28:34.022167+0000","last_fullsized":"2023-10-09T06:28:34.022167+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"173'2174","reported_seq":2189,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:56.757601+0000","last_change":"2023-10-09T06:28:17.434104+0000","last_active":"2023-10-09T06:28:56.757601+0000","last_peered":"2023-10-09T06:28:56.757601+0000","last_clean":"2023-10-09T06:28:56.757601+0000","last_became_active":"2023-10-09T06:28:17.433532+0000","last_became_peered":"2023-10-09T06:28:17.433532+0000","last_unstale":"2023-10-09T06:28:56.757601+0000","last_undegraded":"2023-10-09T06:28:56.757601+0000","last_fullsized":"2023-10-09T06:28:56.757601+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":165,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:28:16.404422+0000","last_clean_scrub_stamp":"2023-10-09T06:28:16.404422+0000","objects_scrubbed":0,"log_size":2174,"log_dups_size":0,"ondisk_log_size":2174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:30.710975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142475264,"num_objects":2174,"num_object_clones":0,"num_object_copies":6522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2174,"num_write_kb":139136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":694,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.041064+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:28:34.041064+0000","last_peered":"2023-10-09T06:28:34.041064+0000","last_clean":"2023-10-09T06:28:34.041064+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:28:34.041064+0000","last_undegraded":"2023-10-09T06:28:34.041064+0000","last_fullsized":"2023-10-09T06:28:34.041064+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024534+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:28:33.024534+0000","last_peered":"2023-10-09T06:28:33.024534+0000","last_clean":"2023-10-09T06:28:33.024534+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:28:33.024534+0000","last_undegraded":"2023-10-09T06:28:33.024534+0000","last_fullsized":"2023-10-09T06:28:33.024534+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038140+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:28:34.038140+0000","last_peered":"2023-10-09T06:28:34.038140+0000","last_clean":"2023-10-09T06:28:34.038140+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:28:34.038140+0000","last_undegraded":"2023-10-09T06:28:34.038140+0000","last_fullsized":"2023-10-09T06:28:34.038140+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":133,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024251+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:28:33.024251+0000","last_peered":"2023-10-09T06:28:33.024251+0000","last_clean":"2023-10-09T06:28:33.024251+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:28:33.024251+0000","last_undegraded":"2023-10-09T06:28:33.024251+0000","last_fullsized":"2023-10-09T06:28:33.024251+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.018520+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:28:33.018520+0000","last_peered":"2023-10-09T06:28:33.018520+0000","last_clean":"2023-10-09T06:28:33.018520+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:28:33.018520+0000","last_undegraded":"2023-10-09T06:28:33.018520+0000","last_fullsized":"2023-10-09T06:28:33.018520+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297716+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:28:34.297716+0000","last_peered":"2023-10-09T06:28:34.297716+0000","last_clean":"2023-10-09T06:28:34.297716+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:28:34.297716+0000","last_undegraded":"2023-10-09T06:28:34.297716+0000","last_fullsized":"2023-10-09T06:28:34.297716+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038454+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:28:34.038454+0000","last_peered":"2023-10-09T06:28:34.038454+0000","last_clean":"2023-10-09T06:28:34.038454+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:28:34.038454+0000","last_undegraded":"2023-10-09T06:28:34.038454+0000","last_fullsized":"2023-10-09T06:28:34.038454+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":152,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.297752+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:28:34.297752+0000","last_peered":"2023-10-09T06:28:34.297752+0000","last_clean":"2023-10-09T06:28:34.297752+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:28:34.297752+0000","last_undegraded":"2023-10-09T06:28:34.297752+0000","last_fullsized":"2023-10-09T06:28:34.297752+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.038067+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:28:34.038067+0000","last_peered":"2023-10-09T06:28:34.038067+0000","last_clean":"2023-10-09T06:28:34.038067+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:28:34.038067+0000","last_undegraded":"2023-10-09T06:28:34.038067+0000","last_fullsized":"2023-10-09T06:28:34.038067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":172,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.023961+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:28:34.023961+0000","last_peered":"2023-10-09T06:28:34.023961+0000","last_clean":"2023-10-09T06:28:34.023961+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:28:34.023961+0000","last_undegraded":"2023-10-09T06:28:34.023961+0000","last_fullsized":"2023-10-09T06:28:34.023961+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":165,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.021440+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:28:33.021440+0000","last_peered":"2023-10-09T06:28:33.021440+0000","last_clean":"2023-10-09T06:28:33.021440+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:28:33.021440+0000","last_undegraded":"2023-10-09T06:28:33.021440+0000","last_fullsized":"2023-10-09T06:28:33.021440+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:28:33.024854+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:28:33.024854+0000","last_peered":"2023-10-09T06:28:33.024854+0000","last_clean":"2023-10-09T06:28:33.024854+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:28:33.024854+0000","last_undegraded":"2023-10-09T06:28:33.024854+0000","last_fullsized":"2023-10-09T06:28:33.024854+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:28:34.037570+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:28:34.037570+0000","last_peered":"2023-10-09T06:28:34.037570+0000","last_clean":"2023-10-09T06:28:34.037570+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:28:34.037570+0000","last_undegraded":"2023-10-09T06:28:34.037570+0000","last_fullsized":"2023-10-09T06:28:34.037570+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2365784064,"num_objects":36099,"num_object_clones":0,"num_object_copies":108297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36099,"num_write_kb":2310336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7196704768,"data_stored":7196704768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36099,"ondisk_log_size":36099,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430572,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069288,"kb_used_data":763872,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92675864,"statfs":{"total":95995035648,"available":94900084736,"internally_reserved":0,"allocated":782204928,"data_stored":781661728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":2.4809999999999999,"15min":1.9379999999999999},"min":{"1min":0.501,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.8460000000000001,"5min":73.491,"15min":107.36799999999999},"last":24.594999999999999},{"interface":"front","average":{"1min":0.79800000000000004,"5min":2.3159999999999998,"15min":1.875},"min":{"1min":0.55300000000000005,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.105,"5min":73.846999999999994,"15min":105.89},"last":24.437000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:28:17 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":2.008,"15min":1.9339999999999999},"min":{"1min":0.39300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":1.7929999999999999,"5min":63.252000000000002,"15min":104.825},"last":0.443},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.9630000000000001,"15min":1.7949999999999999},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.6659999999999999,"5min":60.746000000000002,"15min":104.637},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.742,"15min":2.266},"min":{"1min":0.53100000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.1719999999999999,"5min":12.789999999999999,"15min":104.992},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.5740000000000001,"15min":2.2069999999999999},"min":{"1min":0.51800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.163,"5min":6.2770000000000001,"15min":104.691},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:28:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":3.9430000000000001,"15min":2.1309999999999998},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.7919999999999998,"5min":137.11600000000001,"15min":137.11600000000001},"last":24.719000000000001},{"interface":"front","average":{"1min":1.51,"5min":3.8260000000000001,"15min":2.0430000000000001},"min":{"1min":0.42599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.1849999999999996,"5min":137.14400000000001,"15min":137.14400000000001},"last":24.693000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.90200000000000002,"15min":1.1879999999999999},"min":{"1min":0.629,"5min":0.30599999999999999,"15min":0.29399999999999998},"max":{"1min":1.131,"5min":4.2089999999999996,"15min":11.029999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.996,"5min":1.5489999999999999,"15min":1.4730000000000001},"min":{"1min":0.55600000000000005,"5min":0.307,"15min":0.307},"max":{"1min":1.581,"5min":8.5850000000000009,"15min":8.9800000000000004},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.3280000000000001,"15min":1.4279999999999999},"min":{"1min":0.67800000000000005,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.401,"5min":8.6129999999999995,"15min":11.996},"last":0.57299999999999995},{"interface":"front","average":{"1min":1,"5min":1.419,"15min":1.4059999999999999},"min":{"1min":0.66700000000000004,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":1.2829999999999999,"5min":8.9169999999999998,"15min":13.634},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.157,"15min":1.198},"min":{"1min":0.70799999999999996,"5min":0.379,"15min":0.34399999999999997},"max":{"1min":1.8720000000000001,"5min":6.8019999999999996,"15min":9.9510000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.4990000000000001,"15min":1.4179999999999999},"min":{"1min":0.623,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.55,"5min":8.3889999999999993,"15min":8.6199999999999992},"last":0.66400000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626804,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064152,"kb_used_data":762832,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92681000,"statfs":{"total":95995035648,"available":94905344000,"internally_reserved":0,"allocated":781139968,"data_stored":780612395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.119,"15min":1.242},"min":{"1min":0.34699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.3109999999999999,"5min":8.2520000000000007,"15min":32.289999999999999},"last":1.849},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.6020000000000001,"15min":1.3560000000000001},"min":{"1min":0.499,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":5.4779999999999998,"5min":8.2639999999999993,"15min":31.454999999999998},"last":1.615}]},{"osd":1,"last update":"Mon Oct 9 06:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.3999999999999999,"15min":1.635},"min":{"1min":0.53100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.8929999999999998,"5min":10.331,"15min":137.93899999999999},"last":1.5960000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.4199999999999999,"15min":1.669},"min":{"1min":0.65700000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.9189999999999996,"5min":8.2050000000000001,"15min":138.02500000000001},"last":1.6319999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.7150000000000001,"15min":1.8859999999999999},"min":{"1min":0.64600000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.272,"5min":11.202999999999999,"15min":95.540000000000006},"last":1.8069999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.5249999999999999,"15min":1.847},"min":{"1min":0.69299999999999995,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.5589999999999999,"5min":7.1150000000000002,"15min":96.352000000000004},"last":1.796}]},{"osd":3,"last update":"Mon Oct 9 06:28:56 2023","interfaces":[{"interface":"back","average":{"1min":7.0519999999999996,"5min":3.194,"15min":1.95},"min":{"1min":0.63,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":115.09399999999999,"5min":115.09399999999999,"15min":115.09399999999999},"last":1.6990000000000001},{"interface":"front","average":{"1min":6.8840000000000003,"5min":2.8679999999999999,"15min":1.724},"min":{"1min":0.56399999999999995,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":112.886,"5min":112.886,"15min":112.886},"last":1.768}]},{"osd":4,"last update":"Mon Oct 9 06:28:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.2629999999999999,"15min":1.282},"min":{"1min":0.42799999999999999,"5min":0.31,"15min":0.26000000000000001},"max":{"1min":1.653,"5min":7.3090000000000002,"15min":7.3090000000000002},"last":1.712},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.498,"15min":1.423},"min":{"1min":0.56499999999999995,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8239999999999998,"5min":6.9359999999999999,"15min":10.300000000000001},"last":1.8240000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.615,"15min":1.494},"min":{"1min":0.56200000000000006,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.4399999999999999,"5min":10.138999999999999,"15min":11.754},"last":1.754},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.788,"15min":1.5489999999999999},"min":{"1min":0.68000000000000005,"5min":0.36799999999999999,"15min":0.308},"max":{"1min":3.9089999999999998,"5min":11.026999999999999,"15min":12.295999999999999},"last":1.8620000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:28:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.6679999999999999,"15min":1.514},"min":{"1min":0.50700000000000001,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":5.4160000000000004,"5min":11.076000000000001,"15min":11.076000000000001},"last":1.839},{"interface":"front","average":{"1min":0.998,"5min":1.2410000000000001,"15min":1.4419999999999999},"min":{"1min":0.439,"5min":0.38,"15min":0.38},"max":{"1min":1.8819999999999999,"5min":10.74,"15min":11.102},"last":1.8819999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958713,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1112652,"kb_used_data":844228,"kb_used_omap":7,"kb_used_meta":268408,"kb_avail":92632500,"statfs":{"total":95995035648,"available":94855680000,"internally_reserved":0,"allocated":864489472,"data_stored":863734987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":274850192},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.3899999999999999,"15min":1.1719999999999999},"min":{"1min":0.33300000000000002,"5min":0.189,"15min":0.189},"max":{"1min":7.3479999999999999,"5min":17.059000000000001,"15min":31.969000000000001},"last":72.742999999999995},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.5569999999999999,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.5800000000000001,"5min":14.243,"15min":40.667999999999999},"last":71.962999999999994}]},{"osd":2,"last update":"Mon Oct 9 06:28:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.7370000000000001,"15min":1.2549999999999999},"min":{"1min":0.38300000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.0700000000000001,"5min":12.32,"15min":12.32},"last":67.206000000000003},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.6439999999999999,"15min":1.27},"min":{"1min":0.32000000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.161,"5min":17.542000000000002,"15min":17.542000000000002},"last":67.132000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":5.1859999999999999,"15min":2.4380000000000002},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.252},"max":{"1min":4.1989999999999998,"5min":127.44499999999999,"15min":127.44499999999999},"last":67.174999999999997},{"interface":"front","average":{"1min":0.94199999999999995,"5min":5.367,"15min":2.4119999999999999},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.23200000000000001},"max":{"1min":3.0169999999999999,"5min":123.959,"15min":123.959},"last":67.697000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.4239999999999999,"15min":1.3160000000000001},"min":{"1min":0.41999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.2809999999999997,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.075,"5min":1.5720000000000001,"15min":1.397},"min":{"1min":0.40100000000000002,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":2.9039999999999999,"5min":12.907999999999999,"15min":12.907999999999999},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.524,"15min":1.4239999999999999},"min":{"1min":0.57699999999999996,"5min":0.35799999999999998,"15min":0.32500000000000001},"max":{"1min":2.7010000000000001,"5min":14.170999999999999,"15min":14.170999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.5880000000000001,"15min":1.498},"min":{"1min":0.39500000000000002,"5min":0.372,"15min":0.36599999999999999},"max":{"1min":4.444,"5min":9.2810000000000006,"15min":9.2810000000000006},"last":0.81100000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.5489999999999999,"15min":1.3440000000000001},"min":{"1min":0.57799999999999996,"5min":0.377,"15min":0.36699999999999999},"max":{"1min":2.7389999999999999,"5min":14.212999999999999,"15min":14.212999999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.7789999999999999,"15min":1.4750000000000001},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":4.3440000000000003,"5min":9.5790000000000006,"15min":9.5790000000000006},"last":0.85099999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":2.0129999999999999,"15min":1.7250000000000001},"min":{"1min":0.505,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.4000000000000004,"5min":30.774999999999999,"15min":30.774999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.871,"15min":1.5449999999999999},"min":{"1min":0.498,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":2.883,"5min":30.815000000000001,"15min":30.815000000000001},"last":0.70999999999999996}]}]},{"osd":0,"up_from":88,"seq":377957122239,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1434048,"kb_used_data":1141688,"kb_used_omap":8,"kb_used_meta":292343,"kb_avail":92311104,"statfs":{"total":95995035648,"available":94526570496,"internally_reserved":0,"allocated":1169088512,"data_stored":1168343882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":299359488},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":3.4729999999999999,"15min":1.9390000000000001},"min":{"1min":0.27900000000000003,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":4.4960000000000004,"5min":138.31700000000001,"15min":138.31700000000001},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":3.3700000000000001,"15min":1.879},"min":{"1min":0.27800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.7359999999999998,"5min":144.37,"15min":144.37},"last":1.2050000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.823,"15min":1.3400000000000001},"min":{"1min":0.32800000000000001,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":0.80400000000000005,"5min":28.745000000000001,"15min":28.745000000000001},"last":0.442},{"interface":"front","average":{"1min":0.63500000000000001,"5min":1.974,"15min":1.474},"min":{"1min":0.376,"5min":0.22900000000000001,"15min":0.20200000000000001},"max":{"1min":1.139,"5min":28.146999999999998,"15min":28.146999999999998},"last":0.28699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:28:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":2.6720000000000002,"15min":1.5649999999999999},"min":{"1min":0.28699999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":23.138999999999999,"5min":59.643000000000001,"15min":59.643000000000001},"last":1.4299999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":2.4630000000000001,"15min":1.4319999999999999},"min":{"1min":0.313,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.4180000000000001,"5min":59.762999999999998,"15min":59.762999999999998},"last":0.56599999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:27:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.2849999999999999,"15min":1.3460000000000001},"min":{"1min":0.36899999999999999,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":4.0060000000000002,"5min":8.077,"15min":9.0890000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.704,"15min":1.546},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.0800000000000001,"5min":28.629000000000001,"15min":28.629000000000001},"last":0.72499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:27:56 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":2.0369999999999999,"15min":1.742},"min":{"1min":0.50800000000000001,"5min":0.38200000000000001,"15min":0.35399999999999998},"max":{"1min":2.1160000000000001,"5min":28.574000000000002,"15min":28.574000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.841,"5min":1.75,"15min":1.53},"min":{"1min":0.63,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":9.1199999999999992,"5min":28.061,"15min":28.061},"last":0.627}]},{"osd":6,"last update":"Mon Oct 9 06:27:56 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":1.8069999999999999,"15min":1.458},"min":{"1min":0.46600000000000003,"5min":0.35399999999999998,"15min":0.34599999999999997},"max":{"1min":4.6299999999999999,"5min":27.966999999999999,"15min":27.966999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.2849999999999999,"15min":1.282},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.0609999999999999,"5min":8.1240000000000006,"15min":8.7629999999999999},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:27:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":1.996,"15min":1.6000000000000001},"min":{"1min":0.63300000000000001,"5min":0.41499999999999998,"15min":0.38200000000000001},"max":{"1min":9.1630000000000003,"5min":28.109999999999999,"15min":28.109999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.855,"5min":1.7769999999999999,"15min":1.6279999999999999},"min":{"1min":0.46100000000000002,"5min":0.39600000000000002,"15min":0.38400000000000001},"max":{"1min":5.7789999999999999,"5min":28.074999999999999,"15min":28.074999999999999},"last":0.55200000000000005}]}]},{"osd":2,"up_from":98,"seq":420906795187,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181884,"kb_used_data":841716,"kb_used_omap":11,"kb_used_meta":340148,"kb_avail":92563268,"statfs":{"total":95995035648,"available":94784786432,"internally_reserved":0,"allocated":861917184,"data_stored":861173906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":348312510},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.5580000000000001,"15min":1.4870000000000001},"min":{"1min":0.29799999999999999,"5min":0.19900000000000001,"15min":0.17899999999999999},"max":{"1min":0.90500000000000003,"5min":10.404,"15min":35.015999999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.999,"5min":1.956,"15min":1.6990000000000001},"min":{"1min":0.34499999999999997,"5min":0.19900000000000001,"15min":0.182},"max":{"1min":3.8090000000000002,"5min":11.983000000000001,"15min":36.415999999999997},"last":1.173}]},{"osd":1,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.905,"15min":1.3540000000000001},"min":{"1min":0.44,"5min":0.23200000000000001,"15min":0.20899999999999999},"max":{"1min":1.8560000000000001,"5min":11.571,"15min":11.571},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.84099999999999997,"5min":2.1179999999999999,"15min":1.617},"min":{"1min":0.39600000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.044,"5min":11.943,"15min":11.943},"last":0.60399999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:28:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6790000000000003,"5min":2.7759999999999998,"15min":1.631},"min":{"1min":0.42099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":50.429000000000002,"5min":50.429000000000002,"15min":50.429000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":4.3559999999999999,"5min":3.0750000000000002,"15min":1.6659999999999999},"min":{"1min":0.51800000000000002,"5min":0.21299999999999999,"15min":0.21199999999999999},"max":{"1min":50.441000000000003,"5min":50.441000000000003,"15min":50.441000000000003},"last":1.603}]},{"osd":4,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":2.0209999999999999,"15min":1.867},"min":{"1min":0.65600000000000003,"5min":0.308,"15min":0.308},"max":{"1min":3.008,"5min":12.021000000000001,"15min":19.184000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.8740000000000001,"15min":1.7},"min":{"1min":0.55700000000000005,"5min":0.34200000000000003,"15min":0.33800000000000002},"max":{"1min":1.409,"5min":9.8399999999999999,"15min":29.408000000000001},"last":0.621}]},{"osd":5,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":2.4430000000000001,"15min":1.899},"min":{"1min":0.63700000000000001,"5min":0.33900000000000002,"15min":0.31},"max":{"1min":2.5350000000000001,"5min":12.08,"15min":12.833},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.778,"15min":1.587},"min":{"1min":0.48599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4710000000000001,"5min":8.1300000000000008,"15min":29.545000000000002},"last":0.83799999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.831,"15min":1.4770000000000001},"min":{"1min":0.58999999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.444,"5min":8.0969999999999995,"15min":29.504000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.7310000000000001,"15min":1.524},"min":{"1min":0.46700000000000003,"5min":0.38400000000000001,"15min":0.33600000000000002},"max":{"1min":1.3740000000000001,"5min":7.077,"15min":8.1349999999999998},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.833,"15min":1.623},"min":{"1min":0.47599999999999998,"5min":0.40500000000000003,"15min":0.308},"max":{"1min":1.323,"5min":7.9349999999999996,"15min":8.3309999999999995},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.113,"5min":2.1589999999999998,"15min":1.8540000000000001},"min":{"1min":0.73199999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.5630000000000002,"5min":12.051,"15min":12.051},"last":0.55600000000000005}]}]},{"osd":3,"up_from":103,"seq":442381631661,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1548224,"kb_used_data":1235896,"kb_used_omap":3,"kb_used_meta":312316,"kb_avail":92196928,"statfs":{"total":95995035648,"available":94409654272,"internally_reserved":0,"allocated":1265557504,"data_stored":1264810375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":319812129},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,6,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":3.3900000000000001,"15min":2.0129999999999999},"min":{"1min":0.26900000000000002,"5min":0.17399999999999999,"15min":0.16900000000000001},"max":{"1min":7.218,"5min":147.988,"15min":147.988},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.4099999999999999,"5min":3.0110000000000001,"15min":1.9339999999999999},"min":{"1min":0.313,"5min":0.215,"15min":0.215},"max":{"1min":4.1360000000000001,"5min":147.94200000000001,"15min":147.94200000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.605,"15min":1.1060000000000001},"min":{"1min":0.28199999999999997,"5min":0.191,"15min":0.191},"max":{"1min":8.125,"5min":10.564,"15min":10.564},"last":1.0740000000000001},{"interface":"front","average":{"1min":2.0800000000000001,"5min":2.0899999999999999,"15min":1.3080000000000001},"min":{"1min":0.47499999999999998,"5min":0.218,"15min":0.218},"max":{"1min":8.593,"5min":15.119,"15min":15.119},"last":2.2029999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":2.2530000000000001,"15min":1.7729999999999999},"min":{"1min":0.44500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.138999999999999,"5min":12.951000000000001,"15min":70.594999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.7509999999999999,"15min":1.4510000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.6760000000000002,"5min":8.7940000000000005,"15min":62.991999999999997},"last":1.3999999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.448,"15min":1.393},"min":{"1min":0.42599999999999999,"5min":0.35399999999999998,"15min":0.314},"max":{"1min":3.3410000000000002,"5min":8.3780000000000001,"15min":10.446999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0270000000000001,"15min":1.5609999999999999},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.7469999999999999,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.9259999999999999,"15min":1.4650000000000001},"min":{"1min":0.35999999999999999,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":7.71,"5min":11.669,"15min":11.669},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.2669999999999999,"5min":1.726,"15min":1.516},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34300000000000003},"max":{"1min":6.0640000000000001,"5min":11.279,"15min":11.754},"last":0.745}]},{"osd":6,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8,"5min":1.9379999999999999,"15min":1.3999999999999999},"min":{"1min":0.52100000000000002,"5min":0.36899999999999999,"15min":0.36099999999999999},"max":{"1min":7.1559999999999997,"5min":8.3989999999999991,"15min":8.3989999999999991},"last":1.175},{"interface":"front","average":{"1min":1.54,"5min":1.615,"15min":1.355},"min":{"1min":0.40600000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.7889999999999997,"5min":10.428000000000001,"15min":10.428000000000001},"last":1.8220000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":1.923,"15min":1.4890000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":5.9939999999999998,"5min":18.695,"15min":18.695},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.897,"5min":2.1440000000000001,"15min":1.5980000000000001},"min":{"1min":0.54700000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":7.1870000000000003,"5min":12.193,"15min":12.193},"last":2.1699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019270,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054752,"kb_used_data":745240,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92690400,"statfs":{"total":95995035648,"available":94914969600,"internally_reserved":0,"allocated":763125760,"data_stored":762590138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.232,"15min":1.95},"min":{"1min":0.42199999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.3079999999999998,"5min":17.199000000000002,"15min":84.888000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.7789999999999999,"15min":2.242},"min":{"1min":0.438,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.5339999999999998,"5min":28.978999999999999,"15min":84.912999999999997},"last":0.73599999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":2.9900000000000002,"15min":2.109},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.23999999999999999},"max":{"1min":4.601,"5min":179.47300000000001,"15min":179.47300000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.704,"5min":3.3780000000000001,"15min":2.4740000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.9909999999999997,"5min":179.447,"15min":179.447},"last":1.284}]},{"osd":2,"last update":"Mon Oct 9 06:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.508,"15min":1.9179999999999999},"min":{"1min":0.39600000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.6139999999999999,"5min":17.614000000000001,"15min":61.783999999999999},"last":1.115},{"interface":"front","average":{"1min":0.997,"5min":1.7190000000000001,"15min":2.0630000000000002},"min":{"1min":0.50700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.2850000000000001,"5min":17.439,"15min":61.551000000000002},"last":0.46899999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":2.7999999999999998,"15min":2.0390000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33000000000000002},"max":{"1min":10.512,"5min":91.361999999999995,"15min":91.361999999999995},"last":1.236},{"interface":"front","average":{"1min":1.028,"5min":2.4100000000000001,"15min":1.714},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.313},"max":{"1min":4.9690000000000003,"5min":89.676000000000002,"15min":89.676000000000002},"last":1.2230000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.1719999999999999,"15min":1.3240000000000001},"min":{"1min":0.38,"5min":0.309,"15min":0.309},"max":{"1min":1.1859999999999999,"5min":8.8369999999999997,"15min":12.242000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.458,"15min":1.6599999999999999},"min":{"1min":0.443,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.016,"5min":8.1349999999999998,"15min":20.113},"last":0.68999999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.2889999999999999,"15min":1.5109999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29499999999999998},"max":{"1min":4.0389999999999997,"5min":5.1799999999999997,"15min":18.010000000000002},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.3700000000000001,"15min":1.365},"min":{"1min":0.47499999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.2469999999999999,"5min":11.505000000000001,"15min":13.913},"last":0.374}]},{"osd":7,"last update":"Mon Oct 9 06:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.579,"15min":1.6679999999999999},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.371},"max":{"1min":4.0579999999999998,"5min":5.9279999999999999,"15min":15.294},"last":1.073},{"interface":"front","average":{"1min":1,"5min":1.6779999999999999,"15min":1.708},"min":{"1min":0.52700000000000002,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":2.2149999999999999,"5min":14.896000000000001,"15min":14.896000000000001},"last":0.56499999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823037,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":929356,"kb_used_data":623940,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92815796,"statfs":{"total":95995035648,"available":95043375104,"internally_reserved":0,"allocated":638914560,"data_stored":638394551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.7969999999999999,"15min":2.0859999999999999},"min":{"1min":0.45100000000000001,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":1.335,"5min":45.198999999999998,"15min":69.793999999999997},"last":1.46},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.679,"15min":1.8580000000000001},"min":{"1min":0.58899999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.4380000000000002,"5min":44.470999999999997,"15min":68.873000000000005},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 06:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":3.29,"15min":2.258},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.494,"5min":161.71199999999999,"15min":161.71199999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":3.5579999999999998,"15min":2.4420000000000002},"min":{"1min":0.40100000000000002,"5min":0.30299999999999999,"15min":0.29399999999999998},"max":{"1min":2.5800000000000001,"5min":162.50200000000001,"15min":162.50200000000001},"last":1.641}]},{"osd":2,"last update":"Mon Oct 9 06:28:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.6599999999999999,"15min":1.901},"min":{"1min":0.66300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.202,"5min":7.5270000000000001,"15min":47.207000000000001},"last":6.806},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.4630000000000001,"15min":1.863},"min":{"1min":0.57199999999999995,"5min":0.32500000000000001,"15min":0.30499999999999999},"max":{"1min":1.304,"5min":6.9329999999999998,"15min":59.082999999999998},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:28:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.633,"15min":1.8340000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29199999999999998},"max":{"1min":2.6339999999999999,"5min":66.153999999999996,"15min":66.153999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.0629999999999999,"5min":2.3980000000000001,"15min":1.6899999999999999},"min":{"1min":0.39400000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.996,"5min":65.792000000000002,"15min":65.792000000000002},"last":0.56899999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:28:24 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.7290000000000001,"15min":1.77},"min":{"1min":0.48099999999999998,"5min":0.31900000000000001,"15min":0.317},"max":{"1min":1.409,"5min":8.5459999999999994,"15min":17.667999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.4319999999999999,"15min":1.5529999999999999},"min":{"1min":0.39300000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.417,"5min":7.3410000000000002,"15min":16.181999999999999},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:28:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.7330000000000001,"15min":1.609},"min":{"1min":0.47699999999999998,"5min":0.372,"15min":0.35699999999999998},"max":{"1min":4.7370000000000001,"5min":15.494999999999999,"15min":17.454999999999998},"last":1.766},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.5,"15min":1.5700000000000001},"min":{"1min":0.45200000000000001,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":1.587,"5min":8.9009999999999998,"15min":12.365},"last":0.65400000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:28:44 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.6719999999999999,"15min":1.72},"min":{"1min":0.57699999999999996,"5min":0.47799999999999998,"15min":0.41699999999999998},"max":{"1min":1.6399999999999999,"5min":6.9160000000000004,"15min":13.047000000000001},"last":0.627},{"interface":"front","average":{"1min":1.085,"5min":1.208,"15min":1.506},"min":{"1min":0.42699999999999999,"5min":0.377,"15min":0.36799999999999999},"max":{"1min":1.5620000000000001,"5min":5.1799999999999997,"15min":11.981},"last":0.76400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1171652608,"data_stored":1171652608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":869203968,"data_stored":869203968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":862453760,"data_stored":862453760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1278541824,"data_stored":1278541824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":788004864,"data_stored":788004864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":637992960,"data_stored":637992960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":804126720,"data_stored":804126720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":784728064,"data_stored":784728064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:29:05.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-09T06:29:05.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:05.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 45669 45653 101.888 78.8125 0.00635098 0.00980752 2023-10-09T06:29:05.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:05.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.394 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:05.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:05.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.661 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:05.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:05.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:05.934 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:05.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:05.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:29:06.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:06.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:06.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:06.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 46955 46939 101.146 80.375 0.0125283 0.00988001 2023-10-09T06:29:06.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:06.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:06.311 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:06 smithi155 bash[59495]: cluster 2023-10-09T06:29:05.014932+0000 mgr.y (mgr.25723) 726 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:29:06.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:06.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:06 smithi125 bash[77299]: cluster 2023-10-09T06:29:05.014932+0000 mgr.y (mgr.25723) 726 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:29:06.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:06 smithi125 bash[80002]: cluster 2023-10-09T06:29:05.014932+0000 mgr.y (mgr.25723) 726 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-09T06:29:06.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:06.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:06.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:06.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:06.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:06.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:06.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:06.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:06.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:07.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:07.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:07.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 48210 48194 100.389 78.4375 0.0186661 0.00995361 2023-10-09T06:29:07.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:07.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:07.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:07.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:07.400 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:07.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:07.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:07.664 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:07.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:07.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:29:07.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:07.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:07.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:08.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:08.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:08.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 49431 49415 99.6123 76.3125 0.0180939 0.0100323 2023-10-09T06:29:08.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:08.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:08 smithi155 bash[59495]: cluster 2023-10-09T06:29:07.015478+0000 mgr.y (mgr.25723) 727 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:29:08.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:08.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:08.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:08.317 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:08 smithi125 bash[77299]: cluster 2023-10-09T06:29:07.015478+0000 mgr.y (mgr.25723) 727 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:29:08.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:08 smithi125 bash[80002]: cluster 2023-10-09T06:29:07.015478+0000 mgr.y (mgr.25723) 727 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:29:08.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:08.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:08.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:08.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:08.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:08.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:08.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:08.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:08.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:08.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:08.971 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:08.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:08.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:09.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:09.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:09.165 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 50669 50653 98.9172 77.375 0.00578563 0.0101004 2023-10-09T06:29:09.166 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:09.171 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:09.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:09.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:09.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:09.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:09.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:09.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:09.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:09.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.787 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:09.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:09.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:09 smithi155 bash[59495]: cluster 2023-10-09T06:29:09.016035+0000 mgr.y (mgr.25723) 728 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:29:09.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:09.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:10.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:09 smithi125 bash[77299]: cluster 2023-10-09T06:29:09.016035+0000 mgr.y (mgr.25723) 728 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:29:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:09 smithi125 bash[80002]: cluster 2023-10-09T06:29:09.016035+0000 mgr.y (mgr.25723) 728 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T06:29:10.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:10.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 51871 51855 98.1831 75.125 0.0300869 0.0101774 2023-10-09T06:29:10.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:10.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:10.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:10.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:10.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:10.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:10.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:10.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:10.772 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:10.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:10.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:10.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:10.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:11.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:11.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:11.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 53080 53064 97.5176 75.5625 0.0281688 0.0102482 2023-10-09T06:29:11.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:11.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:11.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:11.372 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:11] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.33.4" 2023-10-09T06:29:11.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:11.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:11.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:11.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:11.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:11.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:11.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:11.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:11.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:11.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:12.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:12.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:12.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:12.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 54266 54250 96.8491 74.125 0.00984856 0.010318 2023-10-09T06:29:12.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:12.219 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:12.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:12.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:12 smithi125 bash[77299]: cluster 2023-10-09T06:29:11.016680+0000 mgr.y (mgr.25723) 729 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:29:12.294 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:12 smithi125 bash[80002]: cluster 2023-10-09T06:29:11.016680+0000 mgr.y (mgr.25723) 729 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:29:12.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:12 smithi155 bash[59495]: cluster 2023-10-09T06:29:11.016680+0000 mgr.y (mgr.25723) 729 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:29:12.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:12.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:12.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:12.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:12.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:12.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:12.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:12.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:12.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:12.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:12.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:12.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:13.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:13.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:13.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 55468 55452 96.2456 75.125 0.0102031 0.0103841 2023-10-09T06:29:13.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:13.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:13.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:13.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:13.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:13.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:13.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:13.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:13.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:13.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:13.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:13.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:13.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:13.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:14.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:14.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:14.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:14.166 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:14 smithi155 bash[59495]: cluster 2023-10-09T06:29:13.017547+0000 mgr.y (mgr.25723) 730 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:29:14.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 56549 56533 95.4704 67.5625 0.0161119 0.0104676 2023-10-09T06:29:14.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:14.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.298 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:14 smithi125 bash[77299]: cluster 2023-10-09T06:29:13.017547+0000 mgr.y (mgr.25723) 730 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:29:14.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:14 smithi125 bash[80002]: cluster 2023-10-09T06:29:13.017547+0000 mgr.y (mgr.25723) 730 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:29:14.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:14.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:14.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:14.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:14.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:14.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:14.708 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:14.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:14.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:14.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:14.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:15.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:15.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:15.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 57698 57682 94.8477 71.8125 0.00687244 0.0105373 2023-10-09T06:29:15.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:15.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:15.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.371 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:15.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:15.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:15.533 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:15.533 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:15.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:15.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:15.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:15.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:15.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:16.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:16.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:16.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:16.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:16.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 58818 58802 94.2105 70 0.00590412 0.0106078 2023-10-09T06:29:16.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:16.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:16.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:16 smithi155 bash[59495]: cluster 2023-10-09T06:29:15.019137+0000 mgr.y (mgr.25723) 731 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T06:29:16.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:16.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:16.420 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:16 smithi125 bash[77299]: cluster 2023-10-09T06:29:15.019137+0000 mgr.y (mgr.25723) 731 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T06:29:16.421 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:16 smithi125 bash[80002]: cluster 2023-10-09T06:29:15.019137+0000 mgr.y (mgr.25723) 731 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-09T06:29:16.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:16.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:29:16.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:16.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:16.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:16.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:16.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:16.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:16.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:16.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:16.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:16.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:17.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:17.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:17.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:29:17.165327+0000 min lat: 0.00135006 max lat: 0.26168 avg lat: 0.0106662 2023-10-09T06:29:17.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:29:17.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 60003 59987 93.7067 74.0625 0.0125202 0.0106662 2023-10-09T06:29:17.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:17.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:17.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:17.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:17.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:17.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:17.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:17.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:17.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:17.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:17.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:17.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:17.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:17.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:18.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:18.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:18.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:18.167 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:18 smithi155 bash[59495]: cluster 2023-10-09T06:29:17.019686+0000 mgr.y (mgr.25723) 732 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:29:18.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 61194 61178 93.2366 74.4375 0.0206264 0.01072 2023-10-09T06:29:18.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:18.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:18.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:18.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:18.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:18 smithi125 bash[77299]: cluster 2023-10-09T06:29:17.019686+0000 mgr.y (mgr.25723) 732 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:29:18.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:18 smithi125 bash[80002]: cluster 2023-10-09T06:29:17.019686+0000 mgr.y (mgr.25723) 732 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:29:18.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:18.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:18.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:18.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:18.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:18.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:18.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:18.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:18.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:18.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:18.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:18.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:19.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.096 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:19 smithi155 bash[59495]: audit 2023-10-09T06:29:18.581824+0000 mon.a (mon.0) 938 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:19.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:19.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 62189 62173 92.4973 62.1875 0.0173237 0.0108054 2023-10-09T06:29:19.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:19.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:19.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.367 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:19 smithi125 bash[77299]: audit 2023-10-09T06:29:18.581824+0000 mon.a (mon.0) 938 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:19.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:19 smithi125 bash[80002]: audit 2023-10-09T06:29:18.581824+0000 mon.a (mon.0) 938 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:19.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:19.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:19.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:19.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:19.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:19.768 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:19.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:19.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:19.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:19.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:20.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:20.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:20.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:20.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:20 smithi125 bash[77299]: cluster 2023-10-09T06:29:19.020601+0000 mgr.y (mgr.25723) 733 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-09T06:29:20.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:20 smithi125 bash[80002]: cluster 2023-10-09T06:29:19.020601+0000 mgr.y (mgr.25723) 733 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-09T06:29:20.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:20.170 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 63357 63341 92.0438 73 0.0107114 0.0108586 2023-10-09T06:29:20.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:20.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:20 smithi155 bash[59495]: cluster 2023-10-09T06:29:19.020601+0000 mgr.y (mgr.25723) 733 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-09T06:29:20.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:20.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:20.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:20.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:20.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:20.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:20.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:20.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:20.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:20.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:20.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:20.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:20.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:21.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:21.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 64370 64354 91.3908 63.3125 0.00499188 0.0109366 2023-10-09T06:29:21.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:21.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:21.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.349 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:21.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:21.465 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:21.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:21] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.33.4" 2023-10-09T06:29:21.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:21.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:21.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:21.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:21.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:21.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:21.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:21.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:21.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:21.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:21.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:22.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:22.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:22.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:22.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:22 smithi155 bash[59495]: cluster 2023-10-09T06:29:21.021296+0000 mgr.y (mgr.25723) 734 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:22.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:22.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 65515 65499 90.9501 71.5625 0.00641763 0.010989 2023-10-09T06:29:22.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:22.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:22.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:22.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:22 smithi125 bash[77299]: cluster 2023-10-09T06:29:21.021296+0000 mgr.y (mgr.25723) 734 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:22.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:22 smithi125 bash[80002]: cluster 2023-10-09T06:29:21.021296+0000 mgr.y (mgr.25723) 734 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:22.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:22.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:22.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:22.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:22.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:22.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:22.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:22.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:23.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:23.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:23.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 66542 66526 90.3682 64.1875 0.00866311 0.0110597 2023-10-09T06:29:23.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:23.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:23.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:23.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:23.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:23.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:23.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:23.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:23.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:23.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:23.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:23.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:23.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:23.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:23.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:23.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:24.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.106 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:24 smithi155 bash[59495]: cluster 2023-10-09T06:29:23.022016+0000 mgr.y (mgr.25723) 735 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:29:24.130 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:24.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 67583 67567 89.8298 65.0625 0.0146691 0.0111261 2023-10-09T06:29:24.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:24.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:24.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:24.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:24 smithi125 bash[77299]: cluster 2023-10-09T06:29:23.022016+0000 mgr.y (mgr.25723) 735 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:29:24.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:24 smithi125 bash[80002]: cluster 2023-10-09T06:29:23.022016+0000 mgr.y (mgr.25723) 735 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:29:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:24.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:24.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:24.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:24.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:24.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:24.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:24.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:24.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:24.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:24.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:25.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:25.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:25.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:25.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 68638 68622 89.332 65.9375 0.0106558 0.0111874 2023-10-09T06:29:25.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:25.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:25.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:25.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:25.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:29:25.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:25.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:25.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:25.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:25.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:25.828 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:29:25.877 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:29:25.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.931 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:29:25.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:25.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:25.978 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:29:26.020 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:29:26.027 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:26.027 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:26.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:26.063 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:29:26.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:26.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:26.102 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:29:26.147 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.155 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:29:26.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 69524 69508 88.6389 55.375 0.0174282 0.0112742 2023-10-09T06:29:26.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:26.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:26.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:26.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[77299]: cluster 2023-10-09T06:29:25.022600+0000 mgr.y (mgr.25723) 736 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:29:26.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[77299]: audit 2023-10-09T06:29:25.469005+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[77299]: audit 2023-10-09T06:29:25.473109+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.125:0/356740217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[77299]: audit 2023-10-09T06:29:25.767258+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.125:0/2054669623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:26.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[80002]: cluster 2023-10-09T06:29:25.022600+0000 mgr.y (mgr.25723) 736 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:29:26.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[80002]: audit 2023-10-09T06:29:25.469005+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[80002]: audit 2023-10-09T06:29:25.473109+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.125:0/356740217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.330 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:26 smithi125 bash[80002]: audit 2023-10-09T06:29:25.767258+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.125:0/2054669623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:26.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:26.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:26 smithi155 bash[59495]: cluster 2023-10-09T06:29:25.022600+0000 mgr.y (mgr.25723) 736 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:29:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:26 smithi155 bash[59495]: audit 2023-10-09T06:29:25.469005+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:26 smithi155 bash[59495]: audit 2023-10-09T06:29:25.473109+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.125:0/356740217' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:29:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:26 smithi155 bash[59495]: audit 2023-10-09T06:29:25.767258+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.125:0/2054669623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:26.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:26.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:26.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:26.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:26.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:26.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:26.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:26.874 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:26.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:26.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:26.981 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:29:27.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:27.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:27.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:27 smithi125 bash[77299]: audit 2023-10-09T06:29:26.145230+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.125:0/76114663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.125 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:27 smithi125 bash[77299]: audit 2023-10-09T06:29:26.146393+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:27.142 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:29:27.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 70573 70557 88.1773 65.5625 0.0302212 0.0113339 2023-10-09T06:29:27.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:27.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:27 smithi155 bash[59495]: audit 2023-10-09T06:29:26.145230+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.125:0/76114663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:27 smithi155 bash[59495]: audit 2023-10-09T06:29:26.146393+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:27.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:27.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:27 smithi125 bash[80002]: audit 2023-10-09T06:29:26.145230+0000 mon.c (mon.2) 75 : audit 1 from='client.? 172.21.15.125:0/76114663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:27 smithi125 bash[80002]: audit 2023-10-09T06:29:26.146393+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:27.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:27.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:27.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:27.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:27.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:29:27.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:27.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:27.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:27.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:27.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:27.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:28.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:28.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:28.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 71469 71453 87.5463 56 0.00317181 0.011387 2023-10-09T06:29:28.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.023193+0000 mgr.y (mgr.25723) 737 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.107041+0000 mon.a (mon.0) 941 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.123774+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: audit 2023-10-09T06:29:27.129713+0000 mon.a (mon.0) 942 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.130033+0000 mon.a (mon.0) 943 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.141740+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.148088+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[80002]: cluster 2023-10-09T06:29:27.150870+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.023193+0000 mgr.y (mgr.25723) 737 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.107041+0000 mon.a (mon.0) 941 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.123774+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: audit 2023-10-09T06:29:27.129713+0000 mon.a (mon.0) 942 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.130033+0000 mon.a (mon.0) 943 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.141740+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.148088+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.194 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:28 smithi125 bash[77299]: cluster 2023-10-09T06:29:27.150870+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:28.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:28.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.023193+0000 mgr.y (mgr.25723) 737 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.107041+0000 mon.a (mon.0) 941 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.123774+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: audit 2023-10-09T06:29:27.129713+0000 mon.a (mon.0) 942 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.130033+0000 mon.a (mon.0) 943 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.141740+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.148088+0000 osd.4 (osd.4) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:28 smithi155 bash[59495]: cluster 2023-10-09T06:29:27.150870+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e175 with expected crc 2023-10-09T06:29:28.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:28.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:28.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:28.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:28.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:28.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:28.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:28.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:28.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:29.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:29.171 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 72472 72456 87.0682 62.6875 0.015362 0.0114799 2023-10-09T06:29:29.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.254 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:29.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:29.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:29.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:29.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:29.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:29.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:29 smithi125 bash[77299]: cluster 2023-10-09T06:29:29.023763+0000 mgr.y (mgr.25723) 738 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:29:29.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:29.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:30.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:30.031 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:29 smithi155 bash[59495]: cluster 2023-10-09T06:29:29.023763+0000 mgr.y (mgr.25723) 738 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:29:30.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:30.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:29 smithi125 bash[80002]: cluster 2023-10-09T06:29:29.023763+0000 mgr.y (mgr.25723) 738 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:29:30.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:30.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 73490 73474 86.6246 63.625 0.0197706 0.011538 2023-10-09T06:29:30.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:30.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:30.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:30.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:30.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:30.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:30.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:30.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:30.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:30.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:30.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:30.953 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:30.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:30.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:31.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:31.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 74488 74472 86.1755 62.375 0.00442424 0.0115981 2023-10-09T06:29:31.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:31.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:31.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:31.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:31.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:29:31.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:29:31.478 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:31] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.33.4" 2023-10-09T06:29:31.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:31.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:31.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:31.800 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:31.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:31.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:31.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:32.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:32.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:32.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 75379 75363 85.6211 55.6875 0.0149944 0.0116741 2023-10-09T06:29:32.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.223 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:32 smithi125 bash[77299]: cluster 2023-10-09T06:29:31.024343+0000 mgr.y (mgr.25723) 739 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:29:32.223 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:32 smithi125 bash[80002]: cluster 2023-10-09T06:29:31.024343+0000 mgr.y (mgr.25723) 739 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:29:32.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:32.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:32.357 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:32 smithi155 bash[59495]: cluster 2023-10-09T06:29:31.024343+0000 mgr.y (mgr.25723) 739 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:29:32.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:32.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:32.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:32.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.662 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:29:32.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:29:32.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:32.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:32.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:32.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:32.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:33.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:33.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:33.172 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 76272 76256 85.0888 55.8125 0.0198045 0.0117454 2023-10-09T06:29:33.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:33.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:33.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:33.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:33.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:33.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:33.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:33.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:33.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:33.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:33.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:33.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:33.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:33.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:34.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:34.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:34.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:34.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:34 smithi155 bash[59495]: cluster 2023-10-09T06:29:33.024977+0000 mgr.y (mgr.25723) 740 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-10-09T06:29:34.123 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:34 smithi155 bash[59495]: audit 2023-10-09T06:29:33.582766+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:34.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:34.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 77112 77096 84.5171 52.5 0.0150359 0.0118264 2023-10-09T06:29:34.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:34.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:34 smithi125 bash[77299]: cluster 2023-10-09T06:29:33.024977+0000 mgr.y (mgr.25723) 740 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-10-09T06:29:34.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:34 smithi125 bash[77299]: audit 2023-10-09T06:29:33.582766+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:34.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:34 smithi125 bash[80002]: cluster 2023-10-09T06:29:33.024977+0000 mgr.y (mgr.25723) 740 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-10-09T06:29:34.283 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:34 smithi125 bash[80002]: audit 2023-10-09T06:29:33.582766+0000 mon.a (mon.0) 944 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:34.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:34.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:34.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:34.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:34.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:34.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:34.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:34.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:34.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:34.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:35.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:35.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:35.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 78052 78036 84.0728 58.75 0.00910318 0.0118883 2023-10-09T06:29:35.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:35.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:35.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:35.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:35.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:35.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:35.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:35.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:35.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:35.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:35.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:35.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:35.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:35.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:36.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:36.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:36.097 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:36 smithi125 bash[77299]: cluster 2023-10-09T06:29:35.025630+0000 mgr.y (mgr.25723) 741 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:29:36.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:36.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 15 78989 78974 83.6414 58.625 0.00715288 0.0119499 2023-10-09T06:29:36.181 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:36 smithi155 bash[59495]: cluster 2023-10-09T06:29:35.025630+0000 mgr.y (mgr.25723) 741 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:29:36.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:36.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:36.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:36.296 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:36.299 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:36.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:36.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:36 smithi125 bash[80002]: cluster 2023-10-09T06:29:35.025630+0000 mgr.y (mgr.25723) 741 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:29:36.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:36.433 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:29:36.433 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:29:36.433 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:29:36.433 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:29:36.433 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:29:36.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:36.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:36.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:36.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:36.714 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:36.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:36.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:36.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:36.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:36.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:37.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:37.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:29:37.167974+0000 min lat: 0.00135006 max lat: 0.26168 avg lat: 0.0120139 2023-10-09T06:29:37.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:29:37.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 79897 79881 83.1921 56.6875 0.0114034 0.0120139 2023-10-09T06:29:37.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.238 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:37.313 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:37.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:37.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:37.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:29:37.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:37.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:37.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:37.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:37.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:37.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:37.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:37.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:38.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:38.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:38.100 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:38 smithi155 bash[59495]: cluster 2023-10-09T06:29:37.026219+0000 mgr.y (mgr.25723) 742 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T06:29:38.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:38.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 80821 80805 82.7749 57.75 0.0356945 0.0120755 2023-10-09T06:29:38.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:38.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:38.298 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:38 smithi125 bash[77299]: cluster 2023-10-09T06:29:37.026219+0000 mgr.y (mgr.25723) 742 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T06:29:38.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:38 smithi125 bash[80002]: cluster 2023-10-09T06:29:37.026219+0000 mgr.y (mgr.25723) 742 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-10-09T06:29:38.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:38.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:38.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:38.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:38.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:38.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:38.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:38.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:38.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:38.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:38.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:38.977 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:39.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:39.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:39.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:39.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 81760 81744 82.3864 58.6875 0.00966452 0.0121324 2023-10-09T06:29:39.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:39.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:39.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:39.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:39.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:39.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:39.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:39.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:39.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:39.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:39.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:39.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:39.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:39.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:39.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:39 smithi125 bash[77299]: cluster 2023-10-09T06:29:39.026684+0000 mgr.y (mgr.25723) 743 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T06:29:39.907 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:39 smithi125 bash[80002]: cluster 2023-10-09T06:29:39.026684+0000 mgr.y (mgr.25723) 743 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T06:29:39.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:39.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:40.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:40.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:40.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:40.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:39 smithi155 bash[59495]: cluster 2023-10-09T06:29:39.026684+0000 mgr.y (mgr.25723) 743 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T06:29:40.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:40.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 82681 82665 81.9923 57.5625 0.0255725 0.0121904 2023-10-09T06:29:40.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:40.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:40.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:40.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:40.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:40.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:40.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:40.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:40.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:40.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:40.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:40.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:40.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:40.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:41.008 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:41.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:41.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 83540 83524 81.55 53.6875 0.0538239 0.012253 2023-10-09T06:29:41.215 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:41] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:29:41.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:41.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:41.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:41.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:41.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:41.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:41.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:41.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:41.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:41.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:41.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:42.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:42.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:42.092 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:42 smithi125 bash[77299]: cluster 2023-10-09T06:29:41.027361+0000 mgr.y (mgr.25723) 744 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:29:42.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 84411 84395 81.1329 54.4375 0.0290185 0.0123192 2023-10-09T06:29:42.190 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:42.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:42.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:42.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:42 smithi125 bash[80002]: cluster 2023-10-09T06:29:41.027361+0000 mgr.y (mgr.25723) 744 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:29:42.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:42 smithi155 bash[59495]: cluster 2023-10-09T06:29:41.027361+0000 mgr.y (mgr.25723) 744 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:29:42.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:42.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:42.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:42.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:42.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:42.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:42.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:42.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:42.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:42.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:43.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:43.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:43.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:43.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 85289 85273 80.735 54.875 0.0169987 0.0123801 2023-10-09T06:29:43.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:43.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:43.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:43.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:43.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:43.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:43.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.643 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:43.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:43.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:43.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:44.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:44.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:44.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:44 smithi125 bash[77299]: cluster 2023-10-09T06:29:43.027966+0000 mgr.y (mgr.25723) 745 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T06:29:44.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:44 smithi125 bash[80002]: cluster 2023-10-09T06:29:43.027966+0000 mgr.y (mgr.25723) 745 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T06:29:44.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:44.173 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 86005 85989 80.1979 44.75 0.00707698 0.0124638 2023-10-09T06:29:44.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:44.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:44.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:44.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:44 smithi155 bash[59495]: cluster 2023-10-09T06:29:43.027966+0000 mgr.y (mgr.25723) 745 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-10-09T06:29:44.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:44.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:44.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:44.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:44.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:44.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:44.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:44.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:44.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:44.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:45.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:45.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:45.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 86911 86895 79.8512 56.625 0.0135364 0.0125174 2023-10-09T06:29:45.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:45.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:45.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:45.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:45.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:45.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:45.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:45.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:45.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:45.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:45.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:46.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:46.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:46.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 87836 87820 79.5317 57.8125 0.00845995 0.0125685 2023-10-09T06:29:46.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:46.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:46.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:46.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:46 smithi125 bash[77299]: cluster 2023-10-09T06:29:45.028776+0000 mgr.y (mgr.25723) 746 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T06:29:46.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:46 smithi125 bash[80002]: cluster 2023-10-09T06:29:45.028776+0000 mgr.y (mgr.25723) 746 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T06:29:46.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:46 smithi155 bash[59495]: cluster 2023-10-09T06:29:45.028776+0000 mgr.y (mgr.25723) 746 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T06:29:46.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:46.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:46.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:46.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:46.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:46.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:46.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:46.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:46.873 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:29:46.873 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:29:46.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:33161 6.8 6:142e2c14:::benchmark_data_smithi125_130224_object33160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:39.633810+0000", 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 367.23404633299998, 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.157963465, 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.874 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 33161 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633810+0000", 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633810+0000", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633809+0000", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013934200000000001 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633948+0000", 2023-10-09T06:29:46.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5789000000000001e-05 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633964+0000", 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0003999999999999e-05 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.633974+0000", 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8016999999999997e-05 2023-10-09T06:29:46.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.634032+0000", 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013557700000000001 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.634168+0000", 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018114100000000001 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.634349+0000", 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016722250000000001 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.636021+0000", 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003616724 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.639638+0000", 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15203369799999999 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.791671+0000", 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5133999999999998e-05 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.791726+0000", 2023-10-09T06:29:46.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7324999999999998e-05 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:39.791774+0000", 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.157963465 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:39514 6.8 6:1216b66a:::benchmark_data_smithi125_130224_object39513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:29:46.881 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:43.964057+0000", 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 362.90379987799997, 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20522101300000001, 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39514 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964057+0000", 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964057+0000", 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964055+0000", 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4066999999999997e-05 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964099+0000", 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9724e-05 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964119+0000", 2023-10-09T06:29:46.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2519999999999999e-06 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964127+0000", 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4166999999999998e-05 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964181+0000", 2023-10-09T06:29:46.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011398 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964295+0000", 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013202900000000001 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.886 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.964427+0000", 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029891990000000001 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.967416+0000", 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036980599999999998 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.967786+0000", 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20122130999999999 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.169007+0000", 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000148677 2023-10-09T06:29:46.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.169156+0000", 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121672 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.169278+0000", 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20522101300000001 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:39518 6.8 6:13d457a8:::benchmark_data_smithi125_130224_object39517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:29:46.890 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:23:43.968092+0000", 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 362.89976462999999, 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20737150400000001, 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 39518 2023-10-09T06:29:46.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968092+0000", 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968092+0000", 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968090+0000", 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8690000000000002e-05 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968149+0000", 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3733999999999999e-05 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968163+0000", 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7360000000000004e-06 2023-10-09T06:29:46.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968169+0000", 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2475000000000002e-05 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968222+0000", 2023-10-09T06:29:46.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014440699999999999 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968366+0000", 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012683599999999999 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.896 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.968493+0000", 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013980069999999999 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.969891+0000", 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0011534679999999999 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:43.971045+0000", 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.204247555 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.175292+0000", 2023-10-09T06:29:46.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117753 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.175410+0000", 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3538999999999997e-05 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:23:44.175464+0000", 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20737150400000001 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117544 6.8 6:1b652c0b:::benchmark_data_smithi125_130224_object6353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.900 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.745727+0000", 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.122129153, 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20021588900000001, 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117544 2023-10-09T06:29:46.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745727+0000", 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745727+0000", 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745726+0000", 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5010000000000004e-06 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745732+0000", 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1878999999999999e-05 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745744+0000", 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6799999999999998e-06 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745750+0000", 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027982520000000002 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.748548+0000", 2023-10-09T06:29:46.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000137491 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.748685+0000", 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8659999999999994e-05 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.748784+0000", 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068350939999999999 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.755619+0000", 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014854305999999999 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770473+0000", 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17534617599999999 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.945820+0000", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9882999999999995e-05 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.945889+0000", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3953000000000003e-05 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.945943+0000", 2023-10-09T06:29:46.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20021588900000001 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117545 6.8 6:14a71af1:::benchmark_data_smithi125_130224_object6354:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.745958+0000", 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.12189834200001, 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20011743900000001, 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.908 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117545 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745958+0000", 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745958+0000", 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745957+0000", 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1080000000000003e-06 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745962+0000", 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0240000000000003e-06 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745971+0000", 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6229999999999996e-06 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.745975+0000", 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0058034080000000004 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.751779+0000", 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010074200000000001 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.751879+0000", 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5235000000000006e-05 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.751975+0000", 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037650829999999998 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.755740+0000", 2023-10-09T06:29:46.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19003742700000001 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.945777+0000", 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021066800000000001 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.945988+0000", 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8526999999999998e-05 2023-10-09T06:29:46.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.946036+0000", 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9441999999999999e-05 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.946076+0000", 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20011743900000001 2023-10-09T06:29:46.915 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117555 6.5 6:aba8c7f1:::benchmark_data_smithi125_130224_object6364:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.761951+0000", 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.10590602600001, 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17877690299999999, 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.916 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117555 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761951+0000", 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761951+0000", 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761948+0000", 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2730000000000008e-06 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761958+0000", 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2413e-05 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761980+0000", 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0815999999999999e-05 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.761991+0000", 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0075999999999999e-05 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762051+0000", 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016253599999999999 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762214+0000", 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012689299999999999 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762341+0000", 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0064435439999999998 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.768784+0000", 2023-10-09T06:29:46.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4685999999999999e-05 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.768829+0000", 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.171715221 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.940544+0000", 2023-10-09T06:29:46.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100894 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.940645+0000", 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2663999999999993e-05 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.940728+0000", 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17877690299999999 2023-10-09T06:29:46.923 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117556 6.5 6:a9194755:::benchmark_data_smithi125_130224_object6365:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.762442+0000", 2023-10-09T06:29:46.924 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.10541503399998, 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18762104700000001, 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117556 2023-10-09T06:29:46.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762442+0000", 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762442+0000", 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:29:46.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762440+0000", 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3849999999999999e-06 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762447+0000", 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2636000000000001e-05 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762460+0000", 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2200000000000003e-06 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.762467+0000", 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002895051 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.765362+0000", 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106502 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.765468+0000", 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9452999999999999e-05 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:46.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.765558+0000", 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034237120000000002 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.768981+0000", 2023-10-09T06:29:46.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015723500000000001 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.769139+0000", 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18082040599999999 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.949959+0000", 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9474e-05 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.950019+0000", 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4085000000000002e-05 2023-10-09T06:29:46.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.950063+0000", 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18762104700000001 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117559 6.8 6:1ae02e27:::benchmark_data_smithi125_130224_object6368:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.766820+0000", 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.10103628100001, 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18312699599999999, 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.934 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117559 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766820+0000", 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766820+0000", 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766819+0000", 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.076e-05 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766829+0000", 2023-10-09T06:29:46.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2751999999999998e-05 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766852+0000", 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0998999999999999e-05 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766863+0000", 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9763999999999999e-05 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.766923+0000", 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118952 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.767042+0000", 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013123599999999999 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.767173+0000", 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032698179999999999 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770443+0000", 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17567922499999999 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.946122+0000", 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003714527 2023-10-09T06:29:46.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.949837+0000", 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1882000000000002e-05 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.949888+0000", 2023-10-09T06:29:46.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8936e-05 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.949947+0000", 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18312699599999999 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:117560 6.8 6:108643bc:::benchmark_data_smithi125_130224_object6369:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:25:01.770632+0000", 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 285.09722439900003, 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17950672000000001, 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.944 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 117560 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770632+0000", 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770632+0000", 2023-10-09T06:29:46.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770631+0000", 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6890000000000004e-06 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770638+0000", 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4771e-05 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770653+0000", 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9429999999999997e-06 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770659+0000", 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8038000000000002e-05 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770687+0000", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4377999999999997e-05 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770772+0000", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6213999999999998e-05 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.770858+0000", 2023-10-09T06:29:46.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026468569999999999 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.773505+0000", 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17629103600000001 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.949796+0000", 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022996 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.950026+0000", 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2434000000000006e-05 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.950088+0000", 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0844999999999999e-05 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.950 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:25:01.950139+0000", 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17950672000000001 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.44939.0:201778 6.8 6:1112a381:::benchmark_data_smithi125_130224_object90587:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:27:00.158424+0000", 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.70943303600001, 2023-10-09T06:29:46.951 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.157474857, 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.44939", 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 201778 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158424+0000", 2023-10-09T06:29:46.952 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158424+0000", 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.953 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158422+0000", 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0369999999999997e-06 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158429+0000", 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1012999999999999e-05 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.954 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158440+0000", 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7319999999999998e-06 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158445+0000", 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026612400000000002 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.955 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158711+0000", 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014798400000000001 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.158859+0000", 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017069200000000001 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.159029+0000", 2023-10-09T06:29:46.956 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0053386459999999998 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.164368+0000", 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019684896 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.184053+0000", 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.131631996 2023-10-09T06:29:46.957 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.315685+0000", 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104986 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.315790+0000", 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108458 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.958 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:27:00.315898+0000", 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.157474857 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:34237 7.5 7:a7abd7df:::benchmark_data_smithi125_153312_object34236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:56.106541+0000", 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.761315824999997, 2023-10-09T06:29:46.959 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.192403878, 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34237 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106541+0000", 2023-10-09T06:29:46.960 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106541+0000", 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106539+0000", 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118187 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.961 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106657+0000", 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9797999999999999e-05 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106677+0000", 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3000000000000002e-06 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.962 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106685+0000", 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8835999999999997e-05 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106744+0000", 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012097200000000001 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.963 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106865+0000", 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013189699999999999 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.106997+0000", 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013538059999999999 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.964 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.108350+0000", 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0076725700000000001 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.116023+0000", 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18272347999999999 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.965 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.298746+0000", 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107178 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.298854+0000", 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1039999999999996e-05 2023-10-09T06:29:46.966 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.298945+0000", 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.192403878 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.967 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:34248 7.5 7:a0b011b3:::benchmark_data_smithi125_153312_object34247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:56.117885+0000", 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.749971635999998, 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.181369538, 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.968 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34248 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117885+0000", 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.969 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117885+0000", 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117883+0000", 2023-10-09T06:29:46.970 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0573000000000003e-05 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117953+0000", 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4199e-05 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.971 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117977+0000", 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2537999999999999e-05 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.117990+0000", 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5790000000000005e-05 2023-10-09T06:29:46.972 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.118056+0000", 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014877099999999999 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.973 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.118205+0000", 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018872299999999999 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.118393+0000", 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015571770000000001 2023-10-09T06:29:46.974 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.119950+0000", 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001931571 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.975 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.121882+0000", 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17723515100000001 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299117+0000", 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3763000000000006e-05 2023-10-09T06:29:46.976 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299181+0000", 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3573999999999997e-05 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.977 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299255+0000", 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.181369538 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:34252 7.5 7:a5297386:::benchmark_data_smithi125_153312_object34251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:56.120073+0000", 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 50.747783292000001, 2023-10-09T06:29:46.978 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.179366043, 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 34252 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.979 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120073+0000", 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120073+0000", 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.980 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120072+0000", 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1315000000000002e-05 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120163+0000", 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9457999999999999e-05 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.981 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120183+0000", 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7219999999999994e-06 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120192+0000", 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8423e-05 2023-10-09T06:29:46.982 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120251+0000", 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000208717 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120459+0000", 2023-10-09T06:29:46.983 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018197800000000001 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.120641+0000", 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013279450000000001 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.984 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.121969+0000", 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00054129099999999995 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.122511+0000", 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17682540099999999 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.985 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299336+0000", 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0309000000000001e-05 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299396+0000", 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3093999999999998e-05 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.986 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:56.299439+0000", 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.179366043 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:41288 7.5 7:abef63ea:::benchmark_data_smithi125_153312_object41287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:01.699835+0000", 2023-10-09T06:29:46.987 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 45.168021553999999, 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13995104899999999, 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 41288 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.988 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699835+0000", 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699835+0000", 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.989 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699834+0000", 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012483199999999999 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699959+0000", 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3304e-05 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.990 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699972+0000", 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6390000000000003e-06 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.699979+0000", 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3755999999999998e-05 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:46.991 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.700032+0000", 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014259400000000001 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.700175+0000", 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014770899999999999 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.700323+0000", 2023-10-09T06:29:46.992 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001565954 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.701889+0000", 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000531213 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.702420+0000", 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.137256301 2023-10-09T06:29:46.993 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.839676+0000", 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4565000000000003e-05 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.839741+0000", 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5385e-05 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.994 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.839786+0000", 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13995104899999999 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:70394 7.5 7:ab250343:::benchmark_data_smithi125_153312_object70393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T06:29:46.995 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:26.979760+0000", 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 19.888096405999999, 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048588514999999999, 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 70394 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:46.996 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979760+0000", 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979760+0000", 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:46.997 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979758+0000", 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0172e-05 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979809+0000", 2023-10-09T06:29:46.998 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6228000000000001e-05 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979825+0000", 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8109999999999997e-06 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979832+0000", 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0417e-05 2023-10-09T06:29:46.999 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.979902+0000", 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151743 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.000 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.980054+0000", 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017898600000000001 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.980233+0000", 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041819530000000004 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.001 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.984415+0000", 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001180501 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:26.985595+0000", 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.042545129000000001 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:27.028140+0000", 2023-10-09T06:29:47.002 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100299 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:27.028241+0000", 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000108099 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:27.028349+0000", 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.048588514999999999 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.003 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:81014 7.5 7:a74b00b7:::benchmark_data_smithi125_153312_object81013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:38.370579+0000", 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 8.4972774809999994, 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060275020999999998, 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:47.004 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 81014 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370579+0000", 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.005 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370579+0000", 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:47.006 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370577+0000", 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9446999999999999e-05 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370667+0000", 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9545e-05 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.007 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370686+0000", 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3250000000000002e-06 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370694+0000", 2023-10-09T06:29:47.008 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1359000000000007e-05 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.370755+0000", 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032195200000000002 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.009 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.371077+0000", 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000164086 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.371241+0000", 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029135469999999998 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.010 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.374155+0000", 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0013503879999999999 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.375505+0000", 2023-10-09T06:29:47.011 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.055164884999999997 2023-10-09T06:29:47.012 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.012 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.012 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.012 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.430670+0000", 2023-10-09T06:29:47.012 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112104 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.430782+0000", 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2212000000000006e-05 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.013 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.430854+0000", 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.060275020999999998 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.014 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:81030 7.5 7:abf3b81e:::benchmark_data_smithi125_153312_object81029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:38.394343+0000", 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 8.4735134090000006, 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050243259999999998, 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:47.015 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 81030 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394343+0000", 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.016 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394343+0000", 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394341+0000", 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115264 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.017 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394456+0000", 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2514000000000002e-05 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394478+0000", 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0023e-05 2023-10-09T06:29:47.018 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394488+0000", 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0146000000000002e-05 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394559+0000", 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000179369 2023-10-09T06:29:47.019 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394738+0000", 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021851499999999999 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.394956+0000", 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003047541 2023-10-09T06:29:47.020 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.398004+0000", 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00055661100000000004 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.398561+0000", 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.045843987000000003 2023-10-09T06:29:47.021 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.444405+0000", 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000101924 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.444507+0000", 2023-10-09T06:29:47.022 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9961000000000002e-05 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:38.444586+0000", 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050243259999999998 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.023 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:83522 7.5 7:a19ab289:::benchmark_data_smithi125_153312_object83521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:41.120771+0000", 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 5.747086049, 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.049556079000000003, 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:47.024 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83522 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120771+0000", 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120771+0000", 2023-10-09T06:29:47.025 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120769+0000", 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8284999999999998e-05 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120867+0000", 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2444e-05 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.026 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120880+0000", 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3019999999999997e-06 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120887+0000", 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8777000000000003e-05 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.027 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.120946+0000", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012911899999999999 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.121075+0000", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013995899999999999 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.121215+0000", 2023-10-09T06:29:47.028 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0027258289999999999 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.123941+0000", 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0019579110000000001 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.125899+0000", 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.044128692999999997 2023-10-09T06:29:47.029 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.170027+0000", 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016778600000000001 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.170195+0000", 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013150400000000001 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.030 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.170327+0000", 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.049556079000000003 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:83530 7.5 7:a36be3e3:::benchmark_data_smithi125_153312_object83529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:29:47.031 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:41.129280+0000", 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 5.7385763179999998, 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050045681000000002, 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83530 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.032 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129280+0000", 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129280+0000", 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.033 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129279+0000", 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3443000000000006e-05 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129373+0000", 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9174999999999999e-05 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:47.034 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129392+0000", 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3130000000000004e-06 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.129401+0000", 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002676809 2023-10-09T06:29:47.035 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.132078+0000", 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018434300000000001 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.132262+0000", 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016770700000000001 2023-10-09T06:29:47.036 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.132430+0000", 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028196639999999999 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.135250+0000", 2023-10-09T06:29:47.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00055938099999999996 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.135809+0000", 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.043377759000000002 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.179187+0000", 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3708000000000006e-05 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.179271+0000", 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5473999999999998e-05 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.179326+0000", 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.050045681000000002 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:83580 7.5 7:a25ab976:::benchmark_data_smithi125_153312_object83579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:41.214400+0000", 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 5.6534567540000005, 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.07181506, 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:29:47.041 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83580 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214400+0000", 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214400+0000", 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214398+0000", 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0173000000000001e-05 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:29:47.043 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214458+0000", 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5064e-05 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214474+0000", 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1160000000000001e-06 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.214481+0000", 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00129857 2023-10-09T06:29:47.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.215779+0000", 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000161974 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.215941+0000", 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021354999999999999 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.216155+0000", 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031299790000000002 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.219285+0000", 2023-10-09T06:29:47.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00064652299999999998 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.219931+0000", 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.066060662000000006 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.285992+0000", 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015871400000000001 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.286151+0000", 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4295999999999996e-05 2023-10-09T06:29:47.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:41.286215+0000", 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.07181506 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.049 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:29:47.050 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:29:47.050 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:29:47.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:47.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:47.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:47 smithi125 bash[77299]: audit 2023-10-09T06:29:47.000917+0000 mon.a (mon.0) 945 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:47.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 88728 88712 79.1919 55.75 0.0129177 0.0126205 2023-10-09T06:29:47.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:47.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:47.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:47.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:47.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:47.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:47 smithi125 bash[80002]: audit 2023-10-09T06:29:47.000917+0000 mon.a (mon.0) 945 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:47.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:47 smithi155 bash[59495]: audit 2023-10-09T06:29:47.000917+0000 mon.a (mon.0) 945 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:47.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:47.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:47.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:47.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:47.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:47.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:47.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:47.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:29:47.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:47.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:47.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:48.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:48.115 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:48 smithi155 bash[59495]: cluster 2023-10-09T06:29:47.029265+0000 mgr.y (mgr.25723) 747 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:29:48.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:48 smithi125 bash[77299]: cluster 2023-10-09T06:29:47.029265+0000 mgr.y (mgr.25723) 747 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:29:48.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:48 smithi125 bash[80002]: cluster 2023-10-09T06:29:47.029265+0000 mgr.y (mgr.25723) 747 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T06:29:48.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 89579 89563 78.8257 53.1875 0.0161256 0.0126806 2023-10-09T06:29:48.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:48.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:48.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:48.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:48.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:48.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:48.652 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:29:48.692 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:29:48.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.739 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:29:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:48.782 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:29:48.823 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:29:48.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:48.863 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:29:48.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:48.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:48.902 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:29:48.942 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.953 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:29:48.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:49.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:49.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 90258 90242 78.3202 42.4375 0.0128554 0.0127626 2023-10-09T06:29:49.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:49.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[77299]: audit 2023-10-09T06:29:48.582255+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:49.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[77299]: audit 2023-10-09T06:29:48.586614+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.125:0/922611360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:49.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[77299]: audit 2023-10-09T06:29:49.033045+0000 mon.a (mon.0) 948 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[77299]: audit 2023-10-09T06:29:49.036073+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.125:0/3172382537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.252 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[77299]: audit 2023-10-09T06:29:49.068558+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:49.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[80002]: audit 2023-10-09T06:29:48.582255+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:49.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[80002]: audit 2023-10-09T06:29:48.586614+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.125:0/922611360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:49.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[80002]: audit 2023-10-09T06:29:49.033045+0000 mon.a (mon.0) 948 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[80002]: audit 2023-10-09T06:29:49.036073+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.125:0/3172382537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.253 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:49 smithi125 bash[80002]: audit 2023-10-09T06:29:49.068558+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:49.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:49.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:49 smithi155 bash[59495]: audit 2023-10-09T06:29:48.582255+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:49 smithi155 bash[59495]: audit 2023-10-09T06:29:48.586614+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.125:0/922611360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:49 smithi155 bash[59495]: audit 2023-10-09T06:29:49.033045+0000 mon.a (mon.0) 948 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:49 smithi155 bash[59495]: audit 2023-10-09T06:29:49.036073+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.125:0/3172382537' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:29:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:49 smithi155 bash[59495]: audit 2023-10-09T06:29:49.068558+0000 mon.a (mon.0) 949 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:49.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:49.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:49.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:49.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:49.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:49.840 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:29:49.840 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:29:49.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:49.925 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:49.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:50.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:50.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:50.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.029816+0000 mgr.y (mgr.25723) 748 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:29:50.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.817249+0000 mon.a (mon.0) 950 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.817282+0000 mon.a (mon.0) 951 : cluster 1 Cluster is now healthy 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: audit 2023-10-09T06:29:49.829047+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: audit 2023-10-09T06:29:49.829285+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.829480+0000 mon.a (mon.0) 954 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.029816+0000 mgr.y (mgr.25723) 748 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.817249+0000 mon.a (mon.0) 950 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:50.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.817282+0000 mon.a (mon.0) 951 : cluster 1 Cluster is now healthy 2023-10-09T06:29:50.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: audit 2023-10-09T06:29:49.829047+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:29:50.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: audit 2023-10-09T06:29:49.829285+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:50.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:50 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.829480+0000 mon.a (mon.0) 954 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:29:50.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:50.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 91138 91122 78.0008 55 0.0281014 0.0128142 2023-10-09T06:29:50.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:50.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:50.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:50.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.029816+0000 mgr.y (mgr.25723) 748 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:29:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.817249+0000 mon.a (mon.0) 950 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.817282+0000 mon.a (mon.0) 951 : cluster 1 Cluster is now healthy 2023-10-09T06:29:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: audit 2023-10-09T06:29:49.829047+0000 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:29:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: audit 2023-10-09T06:29:49.829285+0000 mon.a (mon.0) 953 : audit 1 from='client.? 172.21.15.125:0/329254613' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:50 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.829480+0000 mon.a (mon.0) 954 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:29:50.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:50.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:50.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:50.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:50.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:50.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:50.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:50.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:50.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:50.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:50.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:50.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:51.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:51.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:51.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:51 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.829771+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:51 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.843898+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:51 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.855245+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:51 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.855360+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.119 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:51 smithi155 bash[59495]: cluster 2023-10-09T06:29:49.856623+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:51.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 91978 91962 77.6561 52.5 0.0133539 0.0128719 2023-10-09T06:29:51.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:51.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:51.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:51.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.829771+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.843898+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.855245+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.855360+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.384 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[80002]: cluster 2023-10-09T06:29:49.856623+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.829771+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.843898+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.855245+0000 osd.4 (osd.4) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.855360+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[77299]: cluster 2023-10-09T06:29:49.856623+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e176 with expected crc 2023-10-09T06:29:51.385 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:29:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:29:51] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:29:51.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:51.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:51.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:51.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:51.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:51.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:51.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:51.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:51.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:51.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:52.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:52.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:52.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 92872 92856 77.3657 55.875 0.00888026 0.0129195 2023-10-09T06:29:52.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.233 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:52 smithi125 bash[77299]: cluster 2023-10-09T06:29:51.030357+0000 mgr.y (mgr.25723) 749 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:29:52.233 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:52 smithi125 bash[80002]: cluster 2023-10-09T06:29:51.030357+0000 mgr.y (mgr.25723) 749 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:29:52.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:52.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:52.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:52 smithi155 bash[59495]: cluster 2023-10-09T06:29:51.030357+0000 mgr.y (mgr.25723) 749 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s 2023-10-09T06:29:52.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:52.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:52.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:52.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:52.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:29:52.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:29:52.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:52.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:52.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:53.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:53.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:53.174 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 93675 93659 77.008 50.1875 0.0132298 0.0129787 2023-10-09T06:29:53.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:53.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:53.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:53.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:53.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:53.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:53.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:53.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:53.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:53.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:53.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:53.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:53.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:54.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:54.132 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:54 smithi155 bash[59495]: cluster 2023-10-09T06:29:53.030892+0000 mgr.y (mgr.25723) 750 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T06:29:54.132 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:54 smithi155 bash[59495]: audit 2023-10-09T06:29:53.414427+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:54 smithi155 bash[59495]: audit 2023-10-09T06:29:53.422690+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 94506 94490 76.6824 51.9375 0.0132128 0.0130345 2023-10-09T06:29:54.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:54.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:54.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[77299]: cluster 2023-10-09T06:29:53.030892+0000 mgr.y (mgr.25723) 750 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T06:29:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[77299]: audit 2023-10-09T06:29:53.414427+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[77299]: audit 2023-10-09T06:29:53.422690+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[80002]: cluster 2023-10-09T06:29:53.030892+0000 mgr.y (mgr.25723) 750 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-09T06:29:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[80002]: audit 2023-10-09T06:29:53.414427+0000 mon.a (mon.0) 955 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:54 smithi125 bash[80002]: audit 2023-10-09T06:29:53.422690+0000 mon.a (mon.0) 956 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:54.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:54.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:54.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:54.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:54.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:54.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:29:54.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:54.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:54.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:54.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:54.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:54.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:55.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:55.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:55.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:55.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 95303 95287 76.3379 49.8125 0.0104026 0.0130933 2023-10-09T06:29:55.204 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:55.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:55.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:55.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:55.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:55.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:55.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:55.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:55.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:55.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:55.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:55.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:55.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:56.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:56.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:56 smithi155 bash[59495]: cluster 2023-10-09T06:29:55.031490+0000 mgr.y (mgr.25723) 751 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T06:29:56.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:56.178 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 95995 95979 75.9187 43.25 0.0126163 0.013164 2023-10-09T06:29:56.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:56.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:56.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:56.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:56 smithi125 bash[80002]: cluster 2023-10-09T06:29:55.031490+0000 mgr.y (mgr.25723) 751 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T06:29:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:56 smithi125 bash[77299]: cluster 2023-10-09T06:29:55.031490+0000 mgr.y (mgr.25723) 751 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2023-10-09T06:29:56.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:56.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:56.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:56.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:56.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:56.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:56.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:56.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:56.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:56.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:57.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:57.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:57.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:57.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:29:57.170180+0000 min lat: 0.00135006 max lat: 0.26168 avg lat: 0.0132287 2023-10-09T06:29:57.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:29:57.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 96754 96738 75.5627 47.4375 0.0338003 0.0132287 2023-10-09T06:29:57.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:57.243 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:57.247 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:57.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:57.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:57.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:57.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:57.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:57.520 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:57.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:57.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:57.760 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:57.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: audit 2023-10-09T06:29:56.564551+0000 mon.a (mon.0) 957 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: audit 2023-10-09T06:29:56.575628+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: cluster 2023-10-09T06:29:57.032008+0000 mgr.y (mgr.25723) 752 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: audit 2023-10-09T06:29:57.199067+0000 mon.a (mon.0) 959 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: audit 2023-10-09T06:29:57.200570+0000 mon.a (mon.0) 960 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[77299]: audit 2023-10-09T06:29:57.213213+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: audit 2023-10-09T06:29:56.564551+0000 mon.a (mon.0) 957 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: audit 2023-10-09T06:29:56.575628+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: cluster 2023-10-09T06:29:57.032008+0000 mgr.y (mgr.25723) 752 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:29:57.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: audit 2023-10-09T06:29:57.199067+0000 mon.a (mon.0) 959 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:57.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: audit 2023-10-09T06:29:57.200570+0000 mon.a (mon.0) 960 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:57.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:57 smithi125 bash[80002]: audit 2023-10-09T06:29:57.213213+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: audit 2023-10-09T06:29:56.564551+0000 mon.a (mon.0) 957 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: audit 2023-10-09T06:29:56.575628+0000 mon.a (mon.0) 958 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: cluster 2023-10-09T06:29:57.032008+0000 mgr.y (mgr.25723) 752 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:29:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: audit 2023-10-09T06:29:57.199067+0000 mon.a (mon.0) 959 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: audit 2023-10-09T06:29:57.200570+0000 mon.a (mon.0) 960 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:57 smithi155 bash[59495]: audit 2023-10-09T06:29:57.213213+0000 mon.a (mon.0) 961 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:29:57.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:57.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:57.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:57.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:58.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:58.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:58.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 97592 97576 75.2764 52.375 0.0134006 0.0132782 2023-10-09T06:29:58.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:58.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:58.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:58.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:58.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:58.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:58.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:58.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:58.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:58.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:58.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:58.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:58.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:59.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:59.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:59.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 98437 98421 75.0024 52.8125 0.0100641 0.0133274 2023-10-09T06:29:59.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:59.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:59.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:59.273 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:59.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:59.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:59.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:59.520 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:59.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:59.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:59.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:59.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:59.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:29:59 smithi155 bash[59495]: cluster 2023-10-09T06:29:59.032498+0000 mgr.y (mgr.25723) 753 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:29:59.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:59.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:29:59.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:00.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:00.071 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:29:59 smithi125 bash[77299]: cluster 2023-10-09T06:29:59.032498+0000 mgr.y (mgr.25723) 753 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:30:00.071 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:29:59 smithi125 bash[80002]: cluster 2023-10-09T06:29:59.032498+0000 mgr.y (mgr.25723) 753 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s 2023-10-09T06:30:00.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:00.175 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 99168 99152 74.6492 45.6875 0.0188652 0.01339 2023-10-09T06:30:00.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:00.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:00.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:00.354 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:00.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:00.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:00.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:00.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:00.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:00.759 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:00.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:00.873 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:00 smithi155 bash[59495]: cluster 2023-10-09T06:30:00.000151+0000 mon.a (mon.0) 962 : cluster 1 overall HEALTH_OK 2023-10-09T06:30:00.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:00.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:00.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:00.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:00 smithi125 bash[77299]: cluster 2023-10-09T06:30:00.000151+0000 mon.a (mon.0) 962 : cluster 1 overall HEALTH_OK 2023-10-09T06:30:00.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:00 smithi125 bash[80002]: cluster 2023-10-09T06:30:00.000151+0000 mon.a (mon.0) 962 : cluster 1 overall HEALTH_OK 2023-10-09T06:30:00.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:01.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:01.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:01.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:01.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:01.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 99994 99978 74.375 51.625 0.0106148 0.0134409 2023-10-09T06:30:01.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:01.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:01.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:01.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:01.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:01.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:01.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:01.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:01.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:01] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:30:01.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:01.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:01.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:01.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:01.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:01.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:01.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:01.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:02.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:02.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:01 smithi155 bash[59495]: cluster 2023-10-09T06:30:01.033151+0000 mgr.y (mgr.25723) 754 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T06:30:02.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:02.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:01 smithi125 bash[77299]: cluster 2023-10-09T06:30:01.033151+0000 mgr.y (mgr.25723) 754 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T06:30:02.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:01 smithi125 bash[80002]: cluster 2023-10-09T06:30:01.033151+0000 mgr.y (mgr.25723) 754 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-09T06:30:02.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 100851 100835 74.1302 53.5625 0.0114314 0.0134844 2023-10-09T06:30:02.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:02.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:02.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:02.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:02.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:02.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:02.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:02.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:02.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:02.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:02.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:02.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:02.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:03.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:03.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:03.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 101655 101639 73.8525 50.25 0.0257442 0.0135352 2023-10-09T06:30:03.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:03.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:03.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:03.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.577 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:03.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:03.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:03.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:03.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:03.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:03.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:03.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:03.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:03.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:04.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:04.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:04.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 102414 102398 73.5488 47.4375 0.00406138 0.0135888 2023-10-09T06:30:04.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:04.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:04.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:04 smithi155 bash[59495]: cluster 2023-10-09T06:30:03.033691+0000 mgr.y (mgr.25723) 755 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-09T06:30:04.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:04 smithi155 bash[59495]: audit 2023-10-09T06:30:03.591436+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:04.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:04 smithi155 bash[59495]: audit 2023-10-09T06:30:03.592963+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:04.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[77299]: cluster 2023-10-09T06:30:03.033691+0000 mgr.y (mgr.25723) 755 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-09T06:30:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[77299]: audit 2023-10-09T06:30:03.591436+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:04.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[77299]: audit 2023-10-09T06:30:03.592963+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:04.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[80002]: cluster 2023-10-09T06:30:03.033691+0000 mgr.y (mgr.25723) 755 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-09T06:30:04.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[80002]: audit 2023-10-09T06:30:03.591436+0000 mon.a (mon.0) 963 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:04.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:04 smithi125 bash[80002]: audit 2023-10-09T06:30:03.592963+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:04.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:04.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:04.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:04.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:04.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:04.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:04.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:04.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:04.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:05.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:05.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:05.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:05.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 103124 103108 73.2173 44.375 0.0254572 0.0136528 2023-10-09T06:30:05.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:05.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:05.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:05.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:05.551 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:05.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:05.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:05.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:05.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:05.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:05.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:05.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:05.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:06.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:06.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:06.178 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 103921 103905 72.9543 49.8125 0.0169538 0.0137016 2023-10-09T06:30:06.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:06.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:06 smithi155 bash[59495]: cluster 2023-10-09T06:30:05.034290+0000 mgr.y (mgr.25723) 756 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:30:06.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:06.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:06.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:06.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:06 smithi125 bash[77299]: cluster 2023-10-09T06:30:05.034290+0000 mgr.y (mgr.25723) 756 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:30:06.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:06 smithi125 bash[80002]: cluster 2023-10-09T06:30:05.034290+0000 mgr.y (mgr.25723) 756 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-09T06:30:06.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:06.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:06.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:06.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:06.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:06.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:06.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:06.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:06.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:06.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:06.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:07.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:07.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:07.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.009 2023-10-09T06:30:07.176 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 104684 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 72.69 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 25.979 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 171.375 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 42.4375 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1163 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 415.664 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2742 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 679 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0137524 2023-10-09T06:30:07.177 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0131077 2023-10-09T06:30:07.178 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 0.26168 2023-10-09T06:30:07.178 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00135006 2023-10-09T06:30:07.183 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:30:07.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:07.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:07.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:07.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:07.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:07.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:07.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:07.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:07.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:07.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:07.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:07.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:07.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:07.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:08.109 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:08 smithi125 bash[77299]: cluster 2023-10-09T06:30:07.035052+0000 mgr.y (mgr.25723) 757 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T06:30:08.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:08.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:08.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:08.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:08.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:08 smithi155 bash[59495]: cluster 2023-10-09T06:30:07.035052+0000 mgr.y (mgr.25723) 757 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T06:30:08.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:08.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:08.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:08.404 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:08 smithi125 bash[80002]: cluster 2023-10-09T06:30:07.035052+0000 mgr.y (mgr.25723) 757 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T06:30:08.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:08.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:08.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:08.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:08.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:08.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:08.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:08.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:08.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:08.964 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:30:08.964 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:30:09.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:09.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:09.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:09.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:09.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:09.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:09.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:09.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:09.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:09.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:09.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:09.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:09.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:09.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:09.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:09.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:09.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:10.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:10.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:10.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:09 smithi155 bash[59495]: cluster 2023-10-09T06:30:09.035584+0000 mgr.y (mgr.25723) 758 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 782 op/s 2023-10-09T06:30:10.143 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:09 smithi125 bash[77299]: cluster 2023-10-09T06:30:09.035584+0000 mgr.y (mgr.25723) 758 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 782 op/s 2023-10-09T06:30:10.143 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:09 smithi125 bash[80002]: cluster 2023-10-09T06:30:09.035584+0000 mgr.y (mgr.25723) 758 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 782 op/s 2023-10-09T06:30:10.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:10.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:10.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:10.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:10.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:10.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:10.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:10.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:10.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:10.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:10.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:10.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:10.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:11.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:11.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:11.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:11.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:11.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:11.335 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:11] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:30:11.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:11.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:11.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:11.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:11.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:11.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:11.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:11.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:11.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:11.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:11.902 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:11.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:12.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:12.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:12.096 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:12 smithi155 bash[59495]: cluster 2023-10-09T06:30:11.036267+0000 mgr.y (mgr.25723) 759 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 863 op/s 2023-10-09T06:30:12.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:12.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:12.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:12.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:12.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:12 smithi125 bash[77299]: cluster 2023-10-09T06:30:11.036267+0000 mgr.y (mgr.25723) 759 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 863 op/s 2023-10-09T06:30:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:12 smithi125 bash[80002]: cluster 2023-10-09T06:30:11.036267+0000 mgr.y (mgr.25723) 759 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 863 op/s 2023-10-09T06:30:12.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:12.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:12.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:12.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:12.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:12.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:12.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:12.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:12.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:12.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:12.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:12.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:13.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:13.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:13.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:13.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:13.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:13.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:13.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:13.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:13.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:13.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:13.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:13.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:13.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:13.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:13.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:13.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:30:13.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:13.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:14.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:14.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:14.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:14.188 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:14 smithi155 bash[59495]: cluster 2023-10-09T06:30:13.036766+0000 mgr.y (mgr.25723) 760 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 783 op/s 2023-10-09T06:30:14.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:14.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:14.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:14 smithi125 bash[77299]: cluster 2023-10-09T06:30:13.036766+0000 mgr.y (mgr.25723) 760 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 783 op/s 2023-10-09T06:30:14.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:14 smithi125 bash[80002]: cluster 2023-10-09T06:30:13.036766+0000 mgr.y (mgr.25723) 760 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 783 op/s 2023-10-09T06:30:14.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:14.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:14.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:14.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:14.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:14.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:14.606 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:14.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:14.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:14.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:14.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:14.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:14.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:15.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:15.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:15.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:15.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:15.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:15.407 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:15.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:15.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:15.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:15.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:15.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:15.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.907 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:30:15.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:15.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:15.941 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:30:15.980 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:30:16.025 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:30:16.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:16.064 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:16.068 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:30:16.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:16.109 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:30:16.140 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:30:16.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:16.172 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:30:16.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.177 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.178 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:30:16.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:16.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:16.305 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:16 smithi155 bash[59495]: cluster 2023-10-09T06:30:15.037591+0000 mgr.y (mgr.25723) 761 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 743 op/s 2023-10-09T06:30:16.305 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:16 smithi155 bash[59495]: audit 2023-10-09T06:30:15.778818+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.125:0/1059685261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:16.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:16.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:16 smithi125 bash[77299]: cluster 2023-10-09T06:30:15.037591+0000 mgr.y (mgr.25723) 761 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 743 op/s 2023-10-09T06:30:16.395 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:16 smithi125 bash[77299]: audit 2023-10-09T06:30:15.778818+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.125:0/1059685261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:16.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:16 smithi125 bash[80002]: cluster 2023-10-09T06:30:15.037591+0000 mgr.y (mgr.25723) 761 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 743 op/s 2023-10-09T06:30:16.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:16 smithi125 bash[80002]: audit 2023-10-09T06:30:15.778818+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.125:0/1059685261' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:16.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:16.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:16.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:16.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:16.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:16.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:16.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:16.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:16.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:16.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:16.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:16.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:16.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:17.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:17.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:17.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:17.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:17.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:17.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:17.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:17.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:17.514 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:17.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:17.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:17.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:17.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:17.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:17.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:17.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:18.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:18.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:18.115 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:30:18.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:18.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:18.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:18.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:18.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:18 smithi125 bash[77299]: cluster 2023-10-09T06:30:17.038087+0000 mgr.y (mgr.25723) 762 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 788 op/s 2023-10-09T06:30:18.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:18 smithi125 bash[77299]: audit 2023-10-09T06:30:17.577535+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:18 smithi125 bash[80002]: cluster 2023-10-09T06:30:17.038087+0000 mgr.y (mgr.25723) 762 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 788 op/s 2023-10-09T06:30:18.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:18 smithi125 bash[80002]: audit 2023-10-09T06:30:17.577535+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:18 smithi155 bash[59495]: cluster 2023-10-09T06:30:17.038087+0000 mgr.y (mgr.25723) 762 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 788 op/s 2023-10-09T06:30:18.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:18 smithi155 bash[59495]: audit 2023-10-09T06:30:17.577535+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:18.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:18.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:18.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:18.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:18.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:18.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:18.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:18.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:18.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:18.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:18.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:19.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:19.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: audit 2023-10-09T06:30:18.104828+0000 mon.a (mon.0) 966 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: cluster 2023-10-09T06:30:18.104891+0000 mon.a (mon.0) 967 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: cluster 2023-10-09T06:30:18.107834+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: cluster 2023-10-09T06:30:18.168980+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: cluster 2023-10-09T06:30:18.172014+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: cluster 2023-10-09T06:30:18.174859+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[80002]: audit 2023-10-09T06:30:18.583225+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: audit 2023-10-09T06:30:18.104828+0000 mon.a (mon.0) 966 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: cluster 2023-10-09T06:30:18.104891+0000 mon.a (mon.0) 967 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: cluster 2023-10-09T06:30:18.107834+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: cluster 2023-10-09T06:30:18.168980+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: cluster 2023-10-09T06:30:18.172014+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: cluster 2023-10-09T06:30:18.174859+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.121 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:19 smithi125 bash[77299]: audit 2023-10-09T06:30:18.583225+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:19.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:19.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:19.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:19.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: audit 2023-10-09T06:30:18.104828+0000 mon.a (mon.0) 966 : audit 1 from='client.? 172.21.15.125:0/929657937' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: cluster 2023-10-09T06:30:18.104891+0000 mon.a (mon.0) 967 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: cluster 2023-10-09T06:30:18.107834+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: cluster 2023-10-09T06:30:18.168980+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: cluster 2023-10-09T06:30:18.172014+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: cluster 2023-10-09T06:30:18.174859+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e177 with expected crc 2023-10-09T06:30:19.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:19 smithi155 bash[59495]: audit 2023-10-09T06:30:18.583225+0000 mon.a (mon.0) 968 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:19.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:19.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:19.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:19.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:19.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:19.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:19.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:19.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:19.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:19.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:19.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:19.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:30:20.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:20.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:20.129 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[77299]: cluster 2023-10-09T06:30:19.038417+0000 mgr.y (mgr.25723) 763 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 5.5 MiB/s wr, 794 op/s 2023-10-09T06:30:20.132 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:30:20.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:20.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:20.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:20.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:20 smithi155 bash[59495]: cluster 2023-10-09T06:30:19.038417+0000 mgr.y (mgr.25723) 763 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 5.5 MiB/s wr, 794 op/s 2023-10-09T06:30:20.300 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:20 smithi155 bash[59495]: audit 2023-10-09T06:30:19.116644+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:20 smithi155 bash[59495]: audit 2023-10-09T06:30:19.120714+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.125:0/27165326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:20.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[77299]: audit 2023-10-09T06:30:19.116644+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[77299]: audit 2023-10-09T06:30:19.120714+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.125:0/27165326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[80002]: cluster 2023-10-09T06:30:19.038417+0000 mgr.y (mgr.25723) 763 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 5.5 MiB/s wr, 794 op/s 2023-10-09T06:30:20.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[80002]: audit 2023-10-09T06:30:19.116644+0000 mon.a (mon.0) 969 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:20 smithi125 bash[80002]: audit 2023-10-09T06:30:19.120714+0000 mon.b (mon.1) 15 : audit 1 from='client.? 172.21.15.125:0/27165326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:20.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:20.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:20.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:20.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:20.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:30:20.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:20.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:20.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:20.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:20.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:21.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:21.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:21.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:21.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.112586+0000 mon.a (mon.0) 970 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: audit 2023-10-09T06:30:20.120737+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.120957+0000 mon.a (mon.0) 972 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.138588+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.140646+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.141230+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[77299]: cluster 2023-10-09T06:30:20.144270+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:21] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.112586+0000 mon.a (mon.0) 970 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: audit 2023-10-09T06:30:20.120737+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.120957+0000 mon.a (mon.0) 972 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.138588+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.274 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.140646+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.275 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.141230+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.275 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:21 smithi125 bash[80002]: cluster 2023-10-09T06:30:20.144270+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:21.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.112586+0000 mon.a (mon.0) 970 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:21.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: audit 2023-10-09T06:30:20.120737+0000 mon.a (mon.0) 971 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:21.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.120957+0000 mon.a (mon.0) 972 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:30:21.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.138588+0000 osd.6 (osd.6) 113 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.140646+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.141230+0000 osd.4 (osd.4) 111 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:21 smithi155 bash[59495]: cluster 2023-10-09T06:30:20.144270+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e178 with expected crc 2023-10-09T06:30:21.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:21.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:21.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:21.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:21.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:21.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:21.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:21.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:21.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:21.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:21.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:21.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:22.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:22.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:22.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:22.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:22.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:22.313 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:22 smithi155 bash[59495]: cluster 2023-10-09T06:30:21.038884+0000 mgr.y (mgr.25723) 764 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T06:30:22.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:22.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:22.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:22 smithi125 bash[80002]: cluster 2023-10-09T06:30:21.038884+0000 mgr.y (mgr.25723) 764 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T06:30:22.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:22 smithi125 bash[77299]: cluster 2023-10-09T06:30:21.038884+0000 mgr.y (mgr.25723) 764 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T06:30:22.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:22.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:22.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:22.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:22.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:22.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:22.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:22.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:22.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:22.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:22.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:23.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:23.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:23.170 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:23.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:23.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:23.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:23.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:23.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:30:23.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:23.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:23.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:23.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:23.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:23.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:23.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:23.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:23.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:24.036 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:24.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:24.094 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:24.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:24.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:24.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:24.349 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:24 smithi125 bash[77299]: cluster 2023-10-09T06:30:23.039881+0000 mgr.y (mgr.25723) 765 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-09T06:30:24.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:24 smithi125 bash[80002]: cluster 2023-10-09T06:30:23.039881+0000 mgr.y (mgr.25723) 765 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-09T06:30:24.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:24.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:24.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:24 smithi155 bash[59495]: cluster 2023-10-09T06:30:23.039881+0000 mgr.y (mgr.25723) 765 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-09T06:30:24.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:24.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:24.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:24.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:24.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:24.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:24.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:24.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:24.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:25.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:25.058 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:25.061 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:25.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.064 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:25.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:25.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:25.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:25.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:25.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:25.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:25.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:25.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:25.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:25.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:25.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:25.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:25.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:25.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:25.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:25.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:26.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:26.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:26.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:26.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:26.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:26.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:26.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:26 smithi125 bash[77299]: cluster 2023-10-09T06:30:25.040367+0000 mgr.y (mgr.25723) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:30:26.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:26 smithi125 bash[80002]: cluster 2023-10-09T06:30:25.040367+0000 mgr.y (mgr.25723) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:30:26.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:26.410 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:26 smithi155 bash[59495]: cluster 2023-10-09T06:30:25.040367+0000 mgr.y (mgr.25723) 766 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:30:26.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:26.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:26.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:26.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:26.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:26.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:26.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:26.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:26.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:26.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:27.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:27.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:27.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:27.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:27.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:27.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:27.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:27.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:27.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:27.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:27.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:27.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:27.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:27.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:27.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:27.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:27.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:27.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:28.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:28.111 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:28.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:28.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:28.231 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:28 smithi125 bash[77299]: cluster 2023-10-09T06:30:27.040956+0000 mgr.y (mgr.25723) 767 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:30:28.231 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:28 smithi125 bash[80002]: cluster 2023-10-09T06:30:27.040956+0000 mgr.y (mgr.25723) 767 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:30:28.249 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:28.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:28.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:28.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:28 smithi155 bash[59495]: cluster 2023-10-09T06:30:27.040956+0000 mgr.y (mgr.25723) 767 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:30:28.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:28.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:28.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:28.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:28.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:28.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:28.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:28.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:28.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:28.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:28.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:29.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:29.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:29.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:29.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:29.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:29.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:29.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:29.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:29.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:29.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:29.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:29.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:29.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:30.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:30.120 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:29 smithi155 bash[59495]: cluster 2023-10-09T06:30:29.041532+0000 mgr.y (mgr.25723) 768 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T06:30:30.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:30.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:29 smithi125 bash[77299]: cluster 2023-10-09T06:30:29.041532+0000 mgr.y (mgr.25723) 768 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T06:30:30.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:29 smithi125 bash[80002]: cluster 2023-10-09T06:30:29.041532+0000 mgr.y (mgr.25723) 768 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-10-09T06:30:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:30.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:30.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:30.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:30.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:30.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:30.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:30.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:30.664 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:30.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.798 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:30.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:30.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:31.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:31.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:31.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:31.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:31.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:31.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:31.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:31.578 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:31] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:30:31.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:31.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:31.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:31.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:31.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:31.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:32.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:32.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:32.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:32.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:32.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:32.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:32.195 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:32 smithi125 bash[80002]: cluster 2023-10-09T06:30:31.042194+0000 mgr.y (mgr.25723) 769 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T06:30:32.196 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:32 smithi125 bash[77299]: cluster 2023-10-09T06:30:31.042194+0000 mgr.y (mgr.25723) 769 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T06:30:32.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:32.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:32.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:32 smithi155 bash[59495]: cluster 2023-10-09T06:30:31.042194+0000 mgr.y (mgr.25723) 769 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-09T06:30:32.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:32.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:32.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:32.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:32.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:32.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:32.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:32.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:32.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:32.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:32.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:33.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:33.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:33.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:33.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:33.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:33.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:33.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:33.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:33.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:33.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:34.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:34.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:34.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:34.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[80002]: cluster 2023-10-09T06:30:33.042728+0000 mgr.y (mgr.25723) 770 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T06:30:34.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[80002]: audit 2023-10-09T06:30:33.593456+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:34.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[80002]: audit 2023-10-09T06:30:33.594595+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:34.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[77299]: cluster 2023-10-09T06:30:33.042728+0000 mgr.y (mgr.25723) 770 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T06:30:34.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[77299]: audit 2023-10-09T06:30:33.593456+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:34.283 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:34 smithi125 bash[77299]: audit 2023-10-09T06:30:33.594595+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:34.364 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:34.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:34 smithi155 bash[59495]: cluster 2023-10-09T06:30:33.042728+0000 mgr.y (mgr.25723) 770 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-09T06:30:34.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:34 smithi155 bash[59495]: audit 2023-10-09T06:30:33.593456+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:30:34.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:34 smithi155 bash[59495]: audit 2023-10-09T06:30:33.594595+0000 mon.a (mon.0) 974 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:34.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:34.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:34.633 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:34.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:34.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:34.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:34.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:34.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:34.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:34.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:34.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:35.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:35.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:35.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:35.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:35.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:35.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:35.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:35.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:35.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:35.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:35.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:35.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:35.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:35.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:36.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:36.111 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:36 smithi125 bash[80002]: cluster 2023-10-09T06:30:35.043302+0000 mgr.y (mgr.25723) 771 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:30:36.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:36.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:36.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:36.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:36.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:36 smithi125 bash[77299]: cluster 2023-10-09T06:30:35.043302+0000 mgr.y (mgr.25723) 771 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:30:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:36 smithi155 bash[59495]: cluster 2023-10-09T06:30:35.043302+0000 mgr.y (mgr.25723) 771 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:30:36.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:36.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:36.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:36.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:36.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:36.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:36.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:36.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:36.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:36.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:36.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:36.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:36.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:37.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:37.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:37.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:37.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:37.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:37.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:37.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:37.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:37.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:37.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:37.719 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:37.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:37.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:37.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:37.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:37.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:38.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:38.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:38.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:38 smithi125 bash[77299]: cluster 2023-10-09T06:30:37.043802+0000 mgr.y (mgr.25723) 772 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:30:38.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:38 smithi125 bash[80002]: cluster 2023-10-09T06:30:37.043802+0000 mgr.y (mgr.25723) 772 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:30:38.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:38 smithi155 bash[59495]: cluster 2023-10-09T06:30:37.043802+0000 mgr.y (mgr.25723) 772 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-09T06:30:38.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:38.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:38.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:38.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:38.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:38.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:38.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:38.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:38.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:38.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:38.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:38.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:39.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:39.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:39.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:39.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:39.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:39.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:39.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:39.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:39.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:39.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:39.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:39.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:40.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:40.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:40.106 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:39 smithi125 bash[77299]: cluster 2023-10-09T06:30:39.044381+0000 mgr.y (mgr.25723) 773 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:30:40.106 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:39 smithi125 bash[80002]: cluster 2023-10-09T06:30:39.044381+0000 mgr.y (mgr.25723) 773 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:30:40.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:39 smithi155 bash[59495]: cluster 2023-10-09T06:30:39.044381+0000 mgr.y (mgr.25723) 773 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:30:40.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:40.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:40.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:40.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:40.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:40.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:40.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:40.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:40.611 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:30:40.611 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:30:40.611 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:30:40.611 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:30:40.612 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:30:40.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:40.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:40.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:40.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:40.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:40.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:41.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:41.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:41.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.178 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:41.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:41.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:41.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:41.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:41.423 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:41] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:30:41.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:41.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:41.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:41.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:41.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.711 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:30:41.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:41.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:30:41.793 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:30:41.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.854 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:30:41.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:41.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:41.898 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:30:41.935 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:30:41.973 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:30:41.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:42.000 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.001 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:30:42.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:42.134 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:30:42.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:42.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:42.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:42.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:42.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[77299]: cluster 2023-10-09T06:30:41.045061+0000 mgr.y (mgr.25723) 774 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T06:30:42.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[77299]: audit 2023-10-09T06:30:41.373146+0000 mon.a (mon.0) 975 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:30:42.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[77299]: audit 2023-10-09T06:30:41.599283+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.125:0/1181101065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:42.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[80002]: cluster 2023-10-09T06:30:41.045061+0000 mgr.y (mgr.25723) 774 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T06:30:42.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[80002]: audit 2023-10-09T06:30:41.373146+0000 mon.a (mon.0) 975 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:30:42.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:42 smithi125 bash[80002]: audit 2023-10-09T06:30:41.599283+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.125:0/1181101065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:42 smithi155 bash[59495]: cluster 2023-10-09T06:30:41.045061+0000 mgr.y (mgr.25723) 774 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-10-09T06:30:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:42 smithi155 bash[59495]: audit 2023-10-09T06:30:41.373146+0000 mon.a (mon.0) 975 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:30:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:42 smithi155 bash[59495]: audit 2023-10-09T06:30:41.599283+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.125:0/1181101065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:42.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:42.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:42.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:42.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:42.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:42.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:42.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:42.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:42.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:42.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:42.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:43.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:43.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:43.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:43.232 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:43.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:43.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:43.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:43 smithi155 bash[59495]: cluster 2023-10-09T06:30:42.115607+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:43 smithi155 bash[59495]: audit 2023-10-09T06:30:42.118031+0000 mon.a (mon.0) 976 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:30:43.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:43 smithi155 bash[59495]: cluster 2023-10-09T06:30:42.118157+0000 mon.a (mon.0) 977 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:30:43.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:43 smithi155 bash[59495]: cluster 2023-10-09T06:30:42.127176+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[77299]: cluster 2023-10-09T06:30:42.115607+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[77299]: audit 2023-10-09T06:30:42.118031+0000 mon.a (mon.0) 976 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:30:43.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[77299]: cluster 2023-10-09T06:30:42.118157+0000 mon.a (mon.0) 977 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:30:43.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[77299]: cluster 2023-10-09T06:30:42.127176+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[80002]: cluster 2023-10-09T06:30:42.115607+0000 osd.6 (osd.6) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[80002]: audit 2023-10-09T06:30:42.118031+0000 mon.a (mon.0) 976 : audit 1 from='client.? 172.21.15.125:0/1543233858' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:30:43.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[80002]: cluster 2023-10-09T06:30:42.118157+0000 mon.a (mon.0) 977 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:30:43.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:43 smithi125 bash[80002]: cluster 2023-10-09T06:30:42.127176+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e179 with expected crc 2023-10-09T06:30:43.400 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:43.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:43.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:43.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:43.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:43.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:43.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:43.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:43.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:43.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:43.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:44.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:44.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:44.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:44.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:44.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:44.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:44.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:44 smithi125 bash[77299]: cluster 2023-10-09T06:30:43.045442+0000 mgr.y (mgr.25723) 775 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T06:30:44.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:44 smithi125 bash[80002]: cluster 2023-10-09T06:30:43.045442+0000 mgr.y (mgr.25723) 775 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T06:30:44.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:44 smithi155 bash[59495]: cluster 2023-10-09T06:30:43.045442+0000 mgr.y (mgr.25723) 775 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-09T06:30:44.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:44.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:44.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:44.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:44.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:44.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:44.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:44.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:44.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:45.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:45.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:45.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:45.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:30:45.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:45.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:45.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:45.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:45.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:45.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:45.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:45.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:45.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:45.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:46.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:46.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:46.174 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:30:46.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:46.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:30:46.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[77299]: cluster 2023-10-09T06:30:45.046042+0000 mgr.y (mgr.25723) 776 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:30:46.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[77299]: audit 2023-10-09T06:30:46.073891+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.213 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[77299]: audit 2023-10-09T06:30:46.078077+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.125:0/967185822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.213 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[80002]: cluster 2023-10-09T06:30:45.046042+0000 mgr.y (mgr.25723) 776 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:30:46.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[80002]: audit 2023-10-09T06:30:46.073891+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:46 smithi125 bash[80002]: audit 2023-10-09T06:30:46.078077+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.125:0/967185822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:46.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:46.410 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:46 smithi155 bash[59495]: cluster 2023-10-09T06:30:45.046042+0000 mgr.y (mgr.25723) 776 : cluster 0 pgmap v771: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-09T06:30:46.410 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:46 smithi155 bash[59495]: audit 2023-10-09T06:30:46.073891+0000 mon.a (mon.0) 978 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.410 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:46 smithi155 bash[59495]: audit 2023-10-09T06:30:46.078077+0000 mon.b (mon.1) 16 : audit 1 from='client.? 172.21.15.125:0/967185822' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:46.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:46.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:46.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:46.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:46.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:46.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:46.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:46.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:46.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:46.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:47.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:47.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:47.196 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:47.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:47.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:47.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:47.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[77299]: cluster 2023-10-09T06:30:46.138318+0000 mon.a (mon.0) 979 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:47.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[77299]: audit 2023-10-09T06:30:46.153135+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:47.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[77299]: cluster 2023-10-09T06:30:46.153236+0000 mon.a (mon.0) 981 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[77299]: cluster 2023-10-09T06:30:46.154680+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[77299]: cluster 2023-10-09T06:30:46.155159+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[80002]: cluster 2023-10-09T06:30:46.138318+0000 mon.a (mon.0) 979 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[80002]: audit 2023-10-09T06:30:46.153135+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[80002]: cluster 2023-10-09T06:30:46.153236+0000 mon.a (mon.0) 981 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[80002]: cluster 2023-10-09T06:30:46.154680+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:47 smithi125 bash[80002]: cluster 2023-10-09T06:30:46.155159+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:47.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:47.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:47.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:47 smithi155 bash[59495]: cluster 2023-10-09T06:30:46.138318+0000 mon.a (mon.0) 979 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:47.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:47 smithi155 bash[59495]: audit 2023-10-09T06:30:46.153135+0000 mon.a (mon.0) 980 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:47.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:47 smithi155 bash[59495]: cluster 2023-10-09T06:30:46.153236+0000 mon.a (mon.0) 981 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:30:47.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:47 smithi155 bash[59495]: cluster 2023-10-09T06:30:46.154680+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:47 smithi155 bash[59495]: cluster 2023-10-09T06:30:46.155159+0000 osd.4 (osd.4) 112 : cluster 3 failed to encode map e180 with expected crc 2023-10-09T06:30:47.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:47.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:47.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:47.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:47.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:47.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:47.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:47.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:48.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:48.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:48.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:48.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:30:48.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:48.290 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:48 smithi155 bash[59495]: cluster 2023-10-09T06:30:47.046458+0000 mgr.y (mgr.25723) 777 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:30:48.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:48.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:48.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:48.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:48.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:48.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:48.564 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:48 smithi125 bash[77299]: cluster 2023-10-09T06:30:47.046458+0000 mgr.y (mgr.25723) 777 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:30:48.564 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:48 smithi125 bash[80002]: cluster 2023-10-09T06:30:47.046458+0000 mgr.y (mgr.25723) 777 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:30:48.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:48.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:48.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:48.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:48.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:48.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:48.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:48.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:49.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:49.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:49.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:49.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:49.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:49.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:49.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:30:49.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:49.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:49 smithi125 bash[77299]: audit 2023-10-09T06:30:48.584309+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:49 smithi125 bash[80002]: audit 2023-10-09T06:30:48.584309+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:49.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:49.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:49.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:49 smithi155 bash[59495]: audit 2023-10-09T06:30:48.584309+0000 mon.a (mon.0) 982 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:49.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:49.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:49.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:49.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:49.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:49.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:49.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:49.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:50.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:50.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:50.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:50.181 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:50 smithi125 bash[77299]: cluster 2023-10-09T06:30:49.047041+0000 mgr.y (mgr.25723) 778 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T06:30:50.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:50.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:50.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:50 smithi155 bash[59495]: cluster 2023-10-09T06:30:49.047041+0000 mgr.y (mgr.25723) 778 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T06:30:50.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:50.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:50.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:50 smithi125 bash[80002]: cluster 2023-10-09T06:30:49.047041+0000 mgr.y (mgr.25723) 778 : cluster 0 pgmap v774: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T06:30:50.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:50.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:50.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:50.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:50.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:50.812 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:50.819 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:50.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:50.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:50.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:51.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:51.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:51.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:51.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:51.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:51.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:51.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:51.418 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:30:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:30:51] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-10-09T06:30:51.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:51.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:51.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:51.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:51.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:51.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:51.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:51.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:51.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:51.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:52.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:52.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:52.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:52.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:52.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:52.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:52.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:52 smithi125 bash[77299]: cluster 2023-10-09T06:30:51.047734+0000 mgr.y (mgr.25723) 779 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:30:52.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:52 smithi125 bash[80002]: cluster 2023-10-09T06:30:51.047734+0000 mgr.y (mgr.25723) 779 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:30:52.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:52.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:52.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:52 smithi155 bash[59495]: cluster 2023-10-09T06:30:51.047734+0000 mgr.y (mgr.25723) 779 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:30:52.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:52.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:52.812 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:52.857 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:52.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:52.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:52.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:53.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:53.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:53.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:53.258 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:53.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:53.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:53.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:53.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:53.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:53.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:53.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:53.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:53.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:53.870 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:53.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:53.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:53.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:54.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:54.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:54.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:54.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:54.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:54.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:54 smithi125 bash[77299]: cluster 2023-10-09T06:30:53.048210+0000 mgr.y (mgr.25723) 780 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:30:54.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:54 smithi125 bash[80002]: cluster 2023-10-09T06:30:53.048210+0000 mgr.y (mgr.25723) 780 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:30:54.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:54.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:54.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:54.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:54.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:54.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:54.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:54 smithi155 bash[59495]: cluster 2023-10-09T06:30:53.048210+0000 mgr.y (mgr.25723) 780 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:30:54.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:54.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:54.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:54.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:54.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:54.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:54.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:55.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:55.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:55.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:55.261 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.298 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:55.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:55.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:55.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:55.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:55.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:55.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:55.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:55.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:55.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:56.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:56.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:56.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:56.279 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:56 smithi155 bash[59495]: cluster 2023-10-09T06:30:55.048730+0000 mgr.y (mgr.25723) 781 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:30:56.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:56.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:56.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:56.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:56.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:56 smithi125 bash[77299]: cluster 2023-10-09T06:30:55.048730+0000 mgr.y (mgr.25723) 781 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:30:56.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:56 smithi125 bash[80002]: cluster 2023-10-09T06:30:55.048730+0000 mgr.y (mgr.25723) 781 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:30:56.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:56.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:56.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:56.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:56.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:56.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:56.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:56.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:56.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:57.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:57.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:57.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:57.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:57.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:57.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:57.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:57.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:57.787 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:57.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:57.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:57.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:58.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:58.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:58.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:58.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:58.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:58.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:58.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:58.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:58.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:58 smithi125 bash[77299]: cluster 2023-10-09T06:30:57.049247+0000 mgr.y (mgr.25723) 782 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:30:58.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:58 smithi125 bash[77299]: audit 2023-10-09T06:30:57.327404+0000 mon.a (mon.0) 983 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:58.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:58 smithi125 bash[80002]: cluster 2023-10-09T06:30:57.049247+0000 mgr.y (mgr.25723) 782 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:30:58.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:58 smithi125 bash[80002]: audit 2023-10-09T06:30:57.327404+0000 mon.a (mon.0) 983 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:58.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:58.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:58.581 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:58 smithi155 bash[59495]: cluster 2023-10-09T06:30:57.049247+0000 mgr.y (mgr.25723) 782 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:30:58.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:58 smithi155 bash[59495]: audit 2023-10-09T06:30:57.327404+0000 mon.a (mon.0) 983 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:58.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:58.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:58.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:58.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:58.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:58.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:59.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:59.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:59.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:59.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:59.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:59.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:59.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:59.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:59.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:59.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:59.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:59.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:30:59 smithi125 bash[77299]: cluster 2023-10-09T06:30:59.049772+0000 mgr.y (mgr.25723) 783 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:30:59.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:30:59 smithi125 bash[80002]: cluster 2023-10-09T06:30:59.049772+0000 mgr.y (mgr.25723) 783 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:30:59.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:59.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:59.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:30:59.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:00.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.045 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:30:59 smithi155 bash[59495]: cluster 2023-10-09T06:30:59.049772+0000 mgr.y (mgr.25723) 783 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:31:00.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:00.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:00.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:00.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:00.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:00.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:00.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.481 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:00.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:00.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:00.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:00.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:00.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:00.900 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:00.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:00.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:01.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:01.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:01.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:01.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:01.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:01.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:01.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:01.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:01.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:01.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:01.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:01.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:01.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:01] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.33.4" 2023-10-09T06:31:01.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:01.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:01.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:01.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:01.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:01.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:01.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:01.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:02.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:02.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:02.784 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:02 smithi125 bash[77299]: cluster 2023-10-09T06:31:01.050389+0000 mgr.y (mgr.25723) 784 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T06:31:02.785 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:02 smithi125 bash[80002]: cluster 2023-10-09T06:31:01.050389+0000 mgr.y (mgr.25723) 784 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T06:31:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:02.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:02.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:02 smithi155 bash[59495]: cluster 2023-10-09T06:31:01.050389+0000 mgr.y (mgr.25723) 784 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2023-10-09T06:31:02.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:02.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:02.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:03.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:03.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:03.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:03.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:03.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:03.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:03.488 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:03.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:03.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:03.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:03.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:03.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:03.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:03.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:03.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:04.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:04.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:04.119 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[77299]: cluster 2023-10-09T06:31:03.050853+0000 mgr.y (mgr.25723) 785 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:31:04.119 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[77299]: audit 2023-10-09T06:31:03.587442+0000 mon.a (mon.0) 984 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:04.119 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[77299]: audit 2023-10-09T06:31:03.842892+0000 mon.a (mon.0) 985 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.119 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[77299]: audit 2023-10-09T06:31:03.864903+0000 mon.a (mon.0) 986 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[80002]: cluster 2023-10-09T06:31:03.050853+0000 mgr.y (mgr.25723) 785 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:31:04.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[80002]: audit 2023-10-09T06:31:03.587442+0000 mon.a (mon.0) 984 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:04.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[80002]: audit 2023-10-09T06:31:03.842892+0000 mon.a (mon.0) 985 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.120 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:04 smithi125 bash[80002]: audit 2023-10-09T06:31:03.864903+0000 mon.a (mon.0) 986 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:04.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:04 smithi155 bash[59495]: cluster 2023-10-09T06:31:03.050853+0000 mgr.y (mgr.25723) 785 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:31:04.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:04 smithi155 bash[59495]: audit 2023-10-09T06:31:03.587442+0000 mon.a (mon.0) 984 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:04.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:04 smithi155 bash[59495]: audit 2023-10-09T06:31:03.842892+0000 mon.a (mon.0) 985 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.189 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:04 smithi155 bash[59495]: audit 2023-10-09T06:31:03.864903+0000 mon.a (mon.0) 986 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:04.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:04.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:04.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:04.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:04.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:04.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:04.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:04.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:04.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:04.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:04.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:04.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:04.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:05.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:05.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:05.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:05.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:05.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:05.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:05.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:05.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:05.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:05.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:05.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:05.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:05.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:05.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:05.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:05.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:05.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:06.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:06.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:06.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:06.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:06 smithi155 bash[59495]: cluster 2023-10-09T06:31:05.051451+0000 mgr.y (mgr.25723) 786 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:31:06.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:06.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:06.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:06.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:06 smithi125 bash[77299]: cluster 2023-10-09T06:31:05.051451+0000 mgr.y (mgr.25723) 786 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:31:06.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:06 smithi125 bash[80002]: cluster 2023-10-09T06:31:05.051451+0000 mgr.y (mgr.25723) 786 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-09T06:31:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:06.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:06.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:06.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:06.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:06.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:06.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:06.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:06.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:06.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:07.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:07.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:07.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:07.138 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:31:07.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:07.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:07.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:07 smithi125 bash[77299]: audit 2023-10-09T06:31:06.195333+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.125:0/4020169252' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:07 smithi125 bash[77299]: audit 2023-10-09T06:31:06.196316+0000 mon.a (mon.0) 987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:07 smithi125 bash[80002]: audit 2023-10-09T06:31:06.195333+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.125:0/4020169252' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:07 smithi125 bash[80002]: audit 2023-10-09T06:31:06.196316+0000 mon.a (mon.0) 987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:07.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:07 smithi155 bash[59495]: audit 2023-10-09T06:31:06.195333+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.125:0/4020169252' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:07 smithi155 bash[59495]: audit 2023-10-09T06:31:06.196316+0000 mon.a (mon.0) 987 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:31:07.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:07.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:07.547 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:07.547 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:07.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:07.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:07.681 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:07.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:07.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:07.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:07.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:07.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:08.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:08.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:08.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:08.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:08.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:08.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: cluster 2023-10-09T06:31:07.051944+0000 mgr.y (mgr.25723) 787 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: audit 2023-10-09T06:31:07.121737+0000 mon.a (mon.0) 988 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: cluster 2023-10-09T06:31:07.121806+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: cluster 2023-10-09T06:31:07.123960+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: cluster 2023-10-09T06:31:07.136856+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: audit 2023-10-09T06:31:07.726169+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:08 smithi155 bash[59495]: audit 2023-10-09T06:31:07.735378+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:08.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: cluster 2023-10-09T06:31:07.051944+0000 mgr.y (mgr.25723) 787 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: audit 2023-10-09T06:31:07.121737+0000 mon.a (mon.0) 988 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: cluster 2023-10-09T06:31:07.121806+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: cluster 2023-10-09T06:31:07.123960+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: cluster 2023-10-09T06:31:07.136856+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: audit 2023-10-09T06:31:07.726169+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[77299]: audit 2023-10-09T06:31:07.735378+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: cluster 2023-10-09T06:31:07.051944+0000 mgr.y (mgr.25723) 787 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:31:08.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: audit 2023-10-09T06:31:07.121737+0000 mon.a (mon.0) 988 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:31:08.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: cluster 2023-10-09T06:31:07.121806+0000 mon.a (mon.0) 989 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:31:08.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: cluster 2023-10-09T06:31:07.123960+0000 osd.4 (osd.4) 113 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: cluster 2023-10-09T06:31:07.136856+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-09T06:31:08.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: audit 2023-10-09T06:31:07.726169+0000 mon.a (mon.0) 990 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:08 smithi125 bash[80002]: audit 2023-10-09T06:31:07.735378+0000 mon.a (mon.0) 991 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:08.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:08.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:08.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:08.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:08.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:08.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:08.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:08.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:08.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:09.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:09.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:09.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:09 smithi155 bash[59495]: audit 2023-10-09T06:31:08.312462+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:31:09.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:09 smithi155 bash[59495]: audit 2023-10-09T06:31:08.315702+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:31:09.139 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:09 smithi155 bash[59495]: audit 2023-10-09T06:31:08.325111+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:09.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:09.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:09.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:09.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:09.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[77299]: audit 2023-10-09T06:31:08.312462+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:31:09.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[77299]: audit 2023-10-09T06:31:08.315702+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:31:09.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[77299]: audit 2023-10-09T06:31:08.325111+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:09.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[80002]: audit 2023-10-09T06:31:08.312462+0000 mon.a (mon.0) 992 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:31:09.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[80002]: audit 2023-10-09T06:31:08.315702+0000 mon.a (mon.0) 993 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:31:09.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:09 smithi125 bash[80002]: audit 2023-10-09T06:31:08.325111+0000 mon.a (mon.0) 994 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:09.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:09.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:09.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:09.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:09.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:09.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:09.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:09.800 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:31:09.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:09.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:09.875 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:31:09.923 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:31:09.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:09.960 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:31:09.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:09.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:09.985 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:31:10.015 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:31:10.051 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:31:10.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:10.088 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.096 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.097 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.097 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:31:10.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:10.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:10.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:10 smithi125 bash[77299]: cluster 2023-10-09T06:31:09.052466+0000 mgr.y (mgr.25723) 788 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:31:10.222 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:10 smithi125 bash[77299]: audit 2023-10-09T06:31:09.714751+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.125:0/2678496066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:10.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:10 smithi125 bash[80002]: cluster 2023-10-09T06:31:09.052466+0000 mgr.y (mgr.25723) 788 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:31:10.222 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:10 smithi125 bash[80002]: audit 2023-10-09T06:31:09.714751+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.125:0/2678496066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:10.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:10.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:31:10.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:10.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:10 smithi155 bash[59495]: cluster 2023-10-09T06:31:09.052466+0000 mgr.y (mgr.25723) 788 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:31:10.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:10 smithi155 bash[59495]: audit 2023-10-09T06:31:09.714751+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.125:0/2678496066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:10.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:10.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:10.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:10.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:10.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:10.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:31:10.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:10.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:10.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:10.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:11.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:11.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:11.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:11.161 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:31:11.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:11.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:11.311 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:11.440 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:11.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:11.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:11.575 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:11 smithi125 bash[77299]: audit 2023-10-09T06:31:10.937072+0000 mon.a (mon.0) 995 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:11.575 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:11] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-10-09T06:31:11.575 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:11 smithi125 bash[80002]: audit 2023-10-09T06:31:10.937072+0000 mon.a (mon.0) 995 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:11.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:11.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:11 smithi155 bash[59495]: audit 2023-10-09T06:31:10.937072+0000 mon.a (mon.0) 995 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:11.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:11.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:11.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:11.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:11.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:12.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:12.153 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.052987+0000 mgr.y (mgr.25723) 789 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:31:12.153 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.132421+0000 mon.a (mon.0) 996 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:12.153 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: audit 2023-10-09T06:31:11.147246+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:12.153 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.147304+0000 mon.a (mon.0) 998 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.154262+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.156051+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.159369+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.161472+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.162691+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.155 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[77299]: cluster 2023-10-09T06:31:11.162793+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:12.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.052987+0000 mgr.y (mgr.25723) 789 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.132421+0000 mon.a (mon.0) 996 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: audit 2023-10-09T06:31:11.147246+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.147304+0000 mon.a (mon.0) 998 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.154262+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.156051+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.159369+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.161472+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.162691+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:12 smithi155 bash[59495]: cluster 2023-10-09T06:31:11.162793+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:12.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:12.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:12.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:12.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:12.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.052987+0000 mgr.y (mgr.25723) 789 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.132421+0000 mon.a (mon.0) 996 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: audit 2023-10-09T06:31:11.147246+0000 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.125:0/54161448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.147304+0000 mon.a (mon.0) 998 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.154262+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.156051+0000 osd.4 (osd.4) 114 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.159369+0000 osd.6 (osd.6) 116 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.161472+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.162691+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:12 smithi125 bash[80002]: cluster 2023-10-09T06:31:11.162793+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-09T06:31:12.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:12.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:12.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:12.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:12.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:12.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:12.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:13.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:13.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:13.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:13.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:13.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:13.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:13.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:13.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:13.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:13.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:13.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:13.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:13.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:13.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:13.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:13.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:13.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:13.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:13.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:31:13.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:13.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:13.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:14.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:14.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:14.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:14.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:14 smithi155 bash[59495]: cluster 2023-10-09T06:31:13.053364+0000 mgr.y (mgr.25723) 790 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:31:14.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:14.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:14.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:14 smithi125 bash[77299]: cluster 2023-10-09T06:31:13.053364+0000 mgr.y (mgr.25723) 790 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:31:14.526 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:14 smithi125 bash[80002]: cluster 2023-10-09T06:31:13.053364+0000 mgr.y (mgr.25723) 790 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:31:14.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:14.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:14.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:14.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:31:14.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:14.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:14.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:14.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:15.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:15.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:15.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:15.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:15.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:15.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:15.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:15.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:15.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:15.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:15.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:15.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:15.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:15.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:15.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:15.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:16.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:16.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:16.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:16.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:16.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:16.250 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:16.251 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:16.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:16.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:16 smithi155 bash[59495]: cluster 2023-10-09T06:31:15.054029+0000 mgr.y (mgr.25723) 791 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:31:16.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:16.453 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:16 smithi125 bash[80002]: cluster 2023-10-09T06:31:15.054029+0000 mgr.y (mgr.25723) 791 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:31:16.454 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:16 smithi125 bash[77299]: cluster 2023-10-09T06:31:15.054029+0000 mgr.y (mgr.25723) 791 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:31:16.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:16.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:16.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:16.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:16.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:16.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:16.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:16.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:17.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:17.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:17.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:17.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:17.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:17.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:17.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:17.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:17.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:17.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:17.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:17.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:17.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:17.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:17.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:17.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:18.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:18.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:18.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:18.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:18.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:18.292 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:18.319 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:18 smithi125 bash[77299]: cluster 2023-10-09T06:31:17.054487+0000 mgr.y (mgr.25723) 792 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:18.320 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:18 smithi125 bash[80002]: cluster 2023-10-09T06:31:17.054487+0000 mgr.y (mgr.25723) 792 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:18.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:18.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:18.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:18.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:18.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:18.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:18 smithi155 bash[59495]: cluster 2023-10-09T06:31:17.054487+0000 mgr.y (mgr.25723) 792 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:18.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:18.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:18.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:18.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:18.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:18.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:18.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:18.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:19.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:19.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:19.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:19.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:19.311 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:19.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:19.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:19 smithi125 bash[80002]: audit 2023-10-09T06:31:18.584443+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:19.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:19 smithi125 bash[77299]: audit 2023-10-09T06:31:18.584443+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:19 smithi155 bash[59495]: audit 2023-10-09T06:31:18.584443+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:19.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:19.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:19.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:19.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:19.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:19.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:19.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:19.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:19.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:20.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:20.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:20.232 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:20 smithi155 bash[59495]: cluster 2023-10-09T06:31:19.057125+0000 mgr.y (mgr.25723) 793 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T06:31:20.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:20.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:20.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:20.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:20.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:20 smithi125 bash[77299]: cluster 2023-10-09T06:31:19.057125+0000 mgr.y (mgr.25723) 793 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T06:31:20.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:20 smithi125 bash[80002]: cluster 2023-10-09T06:31:19.057125+0000 mgr.y (mgr.25723) 793 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-09T06:31:20.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.514 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:20.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:20.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:20.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:20.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:20.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:20.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:21.007 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:21.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:21.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:21.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:21.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:21.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:21.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:21.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:21] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-10-09T06:31:21.405 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:21.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:21.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:21.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:21.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:21.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:21.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:21.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:21.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:21.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:21.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:21.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:22.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:22.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:22.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:22.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:22.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:22 smithi125 bash[77299]: cluster 2023-10-09T06:31:21.057745+0000 mgr.y (mgr.25723) 794 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T06:31:22.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:22 smithi125 bash[80002]: cluster 2023-10-09T06:31:21.057745+0000 mgr.y (mgr.25723) 794 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T06:31:22.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:22.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:22.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:22.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:22.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:22.494 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:22 smithi155 bash[59495]: cluster 2023-10-09T06:31:21.057745+0000 mgr.y (mgr.25723) 794 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-10-09T06:31:22.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:22.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:22.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:22.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:22.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:22.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:22.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:22.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:23.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:23.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:23.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:23.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:23.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:23.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:23.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:23.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:23.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:23.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:23.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:23.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:23.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:23.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:23.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:23.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:23.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:24.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:24.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:24.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:24.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:24.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:24.269 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:24 smithi155 bash[59495]: cluster 2023-10-09T06:31:23.058242+0000 mgr.y (mgr.25723) 795 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-09T06:31:24.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:24.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:24.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:24.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:24 smithi125 bash[77299]: cluster 2023-10-09T06:31:23.058242+0000 mgr.y (mgr.25723) 795 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-09T06:31:24.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:24 smithi125 bash[80002]: cluster 2023-10-09T06:31:23.058242+0000 mgr.y (mgr.25723) 795 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-09T06:31:24.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:24.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:24.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:24.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:24.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:24.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:24.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:24.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:24.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:25.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:25.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:25.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:25.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:25.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:25.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:25.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:25.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:25.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:25.598 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:25.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:25.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:25.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:25.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:25.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:25.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:25.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:26.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:26.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:26.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:26.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:26.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:26.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:26.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:26.412 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:26 smithi125 bash[77299]: cluster 2023-10-09T06:31:25.058885+0000 mgr.y (mgr.25723) 796 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:31:26.412 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:26 smithi125 bash[80002]: cluster 2023-10-09T06:31:25.058885+0000 mgr.y (mgr.25723) 796 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:31:26.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:26.450 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:26 smithi155 bash[59495]: cluster 2023-10-09T06:31:25.058885+0000 mgr.y (mgr.25723) 796 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:31:26.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:26.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:26.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:26.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:26.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:26.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:26.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:26.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:26.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:26.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:27.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:27.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:27.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:27.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:27.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:27.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:27.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:27.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:27.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:27.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:27.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:27.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:27.839 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:27.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:27.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:27.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:28.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:28.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:28.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:28.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:28.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:28.369 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:28 smithi155 bash[59495]: cluster 2023-10-09T06:31:27.062885+0000 mgr.y (mgr.25723) 797 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:28.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:28.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:28.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:28.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:28.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:28.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:28 smithi125 bash[77299]: cluster 2023-10-09T06:31:27.062885+0000 mgr.y (mgr.25723) 797 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:28.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:28 smithi125 bash[80002]: cluster 2023-10-09T06:31:27.062885+0000 mgr.y (mgr.25723) 797 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:28.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:28.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:28.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:28.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:28.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:28.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:29.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:29.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:29.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:29.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:29.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:29.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:29.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:29.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:29.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:29.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:29.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:29.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:29.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:29.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:29.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:29.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:29.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:29.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:30.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:30.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:30.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:30.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:29 smithi155 bash[59495]: cluster 2023-10-09T06:31:29.063352+0000 mgr.y (mgr.25723) 798 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:30.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:30.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:29 smithi125 bash[77299]: cluster 2023-10-09T06:31:29.063352+0000 mgr.y (mgr.25723) 798 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:30.248 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:29 smithi125 bash[80002]: cluster 2023-10-09T06:31:29.063352+0000 mgr.y (mgr.25723) 798 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:30.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:30.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:30.411 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:30.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:30.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:30.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:30.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:30.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:30.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:30.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:30.972 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:30.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:31.038 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:31:31.072 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:31 smithi125 bash[77299]: audit 2023-10-09T06:31:30.964533+0000 mon.a (mon.0) 1000 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:31:31.073 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:31 smithi125 bash[80002]: audit 2023-10-09T06:31:30.964533+0000 mon.a (mon.0) 1000 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:31:31.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:31.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:31.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:31.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:31.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:31.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:31.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:31 smithi155 bash[59495]: audit 2023-10-09T06:31:30.964533+0000 mon.a (mon.0) 1000 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:31:31.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:31.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:31] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-10-09T06:31:31.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:31.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:31.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:31.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:31.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:31.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:31.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:31.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:32.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:32.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: audit 2023-10-09T06:31:31.022593+0000 mon.a (mon.0) 1001 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:31:32.035 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.022737+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.024941+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.040446+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.040621+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.043380+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[80002]: cluster 2023-10-09T06:31:31.063775+0000 mgr.y (mgr.25723) 799 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: audit 2023-10-09T06:31:31.022593+0000 mon.a (mon.0) 1001 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.022737+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:31:32.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.024941+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.040446+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.040621+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.043380+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.037 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:32 smithi125 bash[77299]: cluster 2023-10-09T06:31:31.063775+0000 mgr.y (mgr.25723) 799 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:31:32.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:32.160 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:32.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:32.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:32.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:32.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:32.345 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:31:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:31:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.345 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:164036 7.b 7:dcae6e3f:::benchmark_data_smithi125_153312_object59349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:32.333880+0000", 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0040038549999999997, 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040366229999999996, 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:31:32.346 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 164036 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:31:32.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333880+0000", 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333880+0000", 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333879+0000", 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333886+0000", 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1330000000000001e-06 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:31:32.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333902+0000", 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5875e-05 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333910+0000", 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.909e-06 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.333974+0000", 2023-10-09T06:31:32.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4912000000000005e-05 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.334104+0000", 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012936600000000001 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.334223+0000", 2023-10-09T06:31:32.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119252 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:164040 7.b 7.7a3ba4b (undecoded) ondisk+write+known_if_redirected e183)", 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:32.336455+0000", 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001428974, 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001550739, 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:31:32.352 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 164040 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336455+0000", 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336455+0000", 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336453+0000", 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:32.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336462+0000", 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5990000000000002e-06 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336486+0000", 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4219e-05 2023-10-09T06:31:32.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336495+0000", 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7490000000000004e-06 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:164042 7.d 7:b083542b:::benchmark_data_smithi125_153312_object59355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T06:31:32.356 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:32.336507+0000", 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.001376585, 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015659840000000001, 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 164042 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.357 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336507+0000", 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336507+0000", 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336507+0000", 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336510+0000", 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.394e-06 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336518+0000", 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3209999999999999e-06 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336523+0000", 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5500000000000002e-06 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.336803+0000", 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000279747 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.337095+0000", 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000292145 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.337451+0000", 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035612000000000003 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:164029 7.0 e183/165 7:0c5355b1:::benchmark_data_smithi125_153312_object59342:head v 183'10282, mlcod=183'10280)", 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:32.322870+0000", 2023-10-09T06:31:32.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.015014039999999999, 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.015296918, 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322870+0000", 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322870+0000", 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322869+0000", 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:31:32.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322882+0000", 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3548e-05 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322904+0000", 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1327000000000001e-05 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:31:32.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.322913+0000", 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0790000000000007e-06 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.323088+0000", 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017527800000000001 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:32.323146+0000", 2023-10-09T06:31:32.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8353000000000002e-05 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 4 2023-10-09T06:31:32.367 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:31:32.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: audit 2023-10-09T06:31:31.022593+0000 mon.a (mon.0) 1001 : audit 1 from='client.? 172.21.15.125:0/4181127149' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:31:32.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.022737+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:31:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.024941+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.040446+0000 osd.4 (osd.4) 115 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.040621+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.043380+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e183 with expected crc 2023-10-09T06:31:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:32 smithi155 bash[59495]: cluster 2023-10-09T06:31:31.063775+0000 mgr.y (mgr.25723) 799 : cluster 0 pgmap v798: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:31:32.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:32.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.612 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:32.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:32.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:32.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:32.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:32.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:33.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:33.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:33.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:33.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:33.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:33.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:33.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:33.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:33.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:33.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:33.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:33.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:33.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:33.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:33.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:34.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:34.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:34.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:34.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:34.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:34.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:34 smithi125 bash[77299]: cluster 2023-10-09T06:31:33.064306+0000 mgr.y (mgr.25723) 800 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:31:34.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:34 smithi125 bash[77299]: audit 2023-10-09T06:31:33.584784+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:34.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:34 smithi125 bash[80002]: cluster 2023-10-09T06:31:33.064306+0000 mgr.y (mgr.25723) 800 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:31:34.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:34 smithi125 bash[80002]: audit 2023-10-09T06:31:33.584784+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:34 smithi155 bash[59495]: cluster 2023-10-09T06:31:33.064306+0000 mgr.y (mgr.25723) 800 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:31:34.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:34 smithi155 bash[59495]: audit 2023-10-09T06:31:33.584784+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:34.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:34.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:34.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:34.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:34.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:34.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:34.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:35.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:35.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:35.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:35.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:35.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:35.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:35.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:35.457 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:35.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:35.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:35.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:35.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:35.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:35.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:31:35.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:35.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:35.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:36.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:36.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:36.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:36.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:36.284 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:36 smithi125 bash[77299]: cluster 2023-10-09T06:31:35.064919+0000 mgr.y (mgr.25723) 801 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:36.284 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:36 smithi125 bash[80002]: cluster 2023-10-09T06:31:35.064919+0000 mgr.y (mgr.25723) 801 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:36.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:36.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:36.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:36 smithi155 bash[59495]: cluster 2023-10-09T06:31:35.064919+0000 mgr.y (mgr.25723) 801 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:31:36.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:36.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:36.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:36.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:36.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:36.708 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:36.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:36.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:36.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:31:36.944 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:36.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:36.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:37.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.110 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:37.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:37.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:37.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:37.475 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:37.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:37.576 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:31:37.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.619 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:31:37.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:37.668 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:31:37.708 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:31:37.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.750 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:31:37.764 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:37.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:31:37.816 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:31:37.852 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:31:37.859 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.859 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.860 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.860 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.860 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.860 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.860 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.861 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:31:37.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:37.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:38.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:38.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:38.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:38.162 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:31:38.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:38.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:38.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[77299]: cluster 2023-10-09T06:31:37.065480+0000 mgr.y (mgr.25723) 802 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:31:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[77299]: audit 2023-10-09T06:31:37.537238+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.125:0/577954683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[77299]: audit 2023-10-09T06:31:37.899382+0000 mon.a (mon.0) 1005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[77299]: audit 2023-10-09T06:31:37.903327+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.125:0/3919669326' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[80002]: cluster 2023-10-09T06:31:37.065480+0000 mgr.y (mgr.25723) 802 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:31:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[80002]: audit 2023-10-09T06:31:37.537238+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.125:0/577954683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[80002]: audit 2023-10-09T06:31:37.899382+0000 mon.a (mon.0) 1005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:38 smithi125 bash[80002]: audit 2023-10-09T06:31:37.903327+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.125:0/3919669326' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:38 smithi155 bash[59495]: cluster 2023-10-09T06:31:37.065480+0000 mgr.y (mgr.25723) 802 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:31:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:38 smithi155 bash[59495]: audit 2023-10-09T06:31:37.537238+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.125:0/577954683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:38 smithi155 bash[59495]: audit 2023-10-09T06:31:37.899382+0000 mon.a (mon.0) 1005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.417 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:38 smithi155 bash[59495]: audit 2023-10-09T06:31:37.903327+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.125:0/3919669326' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:38.434 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:38.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:38.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:38.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:38.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:38.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:38.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:38.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:38.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:38.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:39.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:39.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:39.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:39.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:39.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:39.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:39.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: cluster 2023-10-09T06:31:38.131856+0000 mon.a (mon.0) 1006 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: cluster 2023-10-09T06:31:38.131895+0000 mon.a (mon.0) 1007 : cluster 1 Cluster is now healthy 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: audit 2023-10-09T06:31:38.141407+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: cluster 2023-10-09T06:31:38.141488+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: cluster 2023-10-09T06:31:38.143202+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.366 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[77299]: cluster 2023-10-09T06:31:38.146959+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: cluster 2023-10-09T06:31:38.131856+0000 mon.a (mon.0) 1006 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: cluster 2023-10-09T06:31:38.131895+0000 mon.a (mon.0) 1007 : cluster 1 Cluster is now healthy 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: audit 2023-10-09T06:31:38.141407+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: cluster 2023-10-09T06:31:38.141488+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: cluster 2023-10-09T06:31:38.143202+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.367 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:39 smithi125 bash[80002]: cluster 2023-10-09T06:31:38.146959+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.371 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:39.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: cluster 2023-10-09T06:31:38.131856+0000 mon.a (mon.0) 1006 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: cluster 2023-10-09T06:31:38.131895+0000 mon.a (mon.0) 1007 : cluster 1 Cluster is now healthy 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: audit 2023-10-09T06:31:38.141407+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: cluster 2023-10-09T06:31:38.141488+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: cluster 2023-10-09T06:31:38.143202+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:39 smithi155 bash[59495]: cluster 2023-10-09T06:31:38.146959+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e184 with expected crc 2023-10-09T06:31:39.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:39.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:39.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:39.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:39.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:39.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:39.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:39.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:39.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:40.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:40.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:40.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:40.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.301 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:40 smithi155 bash[59495]: cluster 2023-10-09T06:31:39.065976+0000 mgr.y (mgr.25723) 803 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:31:40.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:40.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:40.550 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:40.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:40 smithi125 bash[77299]: cluster 2023-10-09T06:31:39.065976+0000 mgr.y (mgr.25723) 803 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:31:40.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:40 smithi125 bash[80002]: cluster 2023-10-09T06:31:39.065976+0000 mgr.y (mgr.25723) 803 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:31:40.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:40.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:40.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:40.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:40.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:40.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:40.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:40.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:40.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:40.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:41.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:41.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:41.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:41.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:41] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:31:41.411 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:41.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:41.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:41.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:41.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:41.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:41.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:31:41.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:41.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:41.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:41.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:42.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:42.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:42.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:42.176 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:42 smithi125 bash[77299]: cluster 2023-10-09T06:31:41.066705+0000 mgr.y (mgr.25723) 804 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:31:42.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:42.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:42.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:42.319 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:42 smithi155 bash[59495]: cluster 2023-10-09T06:31:41.066705+0000 mgr.y (mgr.25723) 804 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:31:42.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:42.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:42.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:42.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:42.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:42.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:42 smithi125 bash[80002]: cluster 2023-10-09T06:31:41.066705+0000 mgr.y (mgr.25723) 804 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:31:42.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:31:42.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:42.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:42.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:42.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:42.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:42.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:42.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:43.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:43.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:43.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:43.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:43.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:43.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:43.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:43.547 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:43.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:43.651 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:43.660 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:43.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:43.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:43.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:43.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:43.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:43.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:43.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:43.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:44.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:44.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:44.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:44.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:44.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:44.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:44 smithi155 bash[59495]: cluster 2023-10-09T06:31:43.067271+0000 mgr.y (mgr.25723) 805 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:31:44.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:44.480 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:44.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:44.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:44.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:44.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:44 smithi125 bash[77299]: cluster 2023-10-09T06:31:43.067271+0000 mgr.y (mgr.25723) 805 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:31:44.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:44 smithi125 bash[80002]: cluster 2023-10-09T06:31:43.067271+0000 mgr.y (mgr.25723) 805 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:31:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:44.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:44.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:44.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:44.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:44.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:45.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:45.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:45.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:45.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:45.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:45.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:45.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:45.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:45.589 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:45.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:45.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:45.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:45.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:45.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:46.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:46.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:46.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:46.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:46.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:46.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:46 smithi125 bash[77299]: cluster 2023-10-09T06:31:45.067903+0000 mgr.y (mgr.25723) 806 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:46.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:46 smithi125 bash[80002]: cluster 2023-10-09T06:31:45.067903+0000 mgr.y (mgr.25723) 806 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:46.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:46.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:46.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:46.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:46 smithi155 bash[59495]: cluster 2023-10-09T06:31:45.067903+0000 mgr.y (mgr.25723) 806 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:31:46.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:46.714 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:46.715 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:46.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:46.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:46.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:46.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:47.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:47.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:47.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:47.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:47.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:47.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:47.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:47.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:47.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:47.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:47.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:47.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:47.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:48.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:48.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:48.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:48.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:48.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:48.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:48 smithi155 bash[59495]: cluster 2023-10-09T06:31:47.068403+0000 mgr.y (mgr.25723) 807 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:31:48.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:48.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:48.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:48.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:48 smithi125 bash[77299]: cluster 2023-10-09T06:31:47.068403+0000 mgr.y (mgr.25723) 807 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:31:48.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:48 smithi125 bash[80002]: cluster 2023-10-09T06:31:47.068403+0000 mgr.y (mgr.25723) 807 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:31:48.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:48.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:48.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:48.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:48.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:48.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:48.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:49.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:49.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:49.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:49.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:49.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:49.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:49.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:49.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:49.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:49.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:49.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:49.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:49.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:49.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:49.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:49.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:49.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:49.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:49.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:49 smithi155 bash[59495]: audit 2023-10-09T06:31:48.596334+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:49 smithi155 bash[59495]: audit 2023-10-09T06:31:48.598100+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:49 smithi155 bash[59495]: cluster 2023-10-09T06:31:49.068910+0000 mgr.y (mgr.25723) 808 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:31:49.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[80002]: audit 2023-10-09T06:31:48.596334+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:49.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[80002]: audit 2023-10-09T06:31:48.598100+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:49.898 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[80002]: cluster 2023-10-09T06:31:49.068910+0000 mgr.y (mgr.25723) 808 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:31:49.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[77299]: audit 2023-10-09T06:31:48.596334+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:31:49.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[77299]: audit 2023-10-09T06:31:48.598100+0000 mon.a (mon.0) 1011 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:49.899 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:49 smithi125 bash[77299]: cluster 2023-10-09T06:31:49.068910+0000 mgr.y (mgr.25723) 808 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T06:31:49.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:50.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:50.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:50.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:50.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:50.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:50.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:50.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:50.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:50.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:50.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:50.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:50.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:50.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:51.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:51.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:51.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:51.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:51.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:51.348 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:31:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:31:51] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:31:51.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:51.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:51.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:51.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:51.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:51.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:51.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:51.748 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:51.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:51.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:51.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:51.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:52.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:52.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:52.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:52.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:52.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:52.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:52 smithi155 bash[59495]: cluster 2023-10-09T06:31:51.069499+0000 mgr.y (mgr.25723) 809 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:31:52.399 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:52 smithi125 bash[77299]: cluster 2023-10-09T06:31:51.069499+0000 mgr.y (mgr.25723) 809 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:31:52.399 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:52 smithi125 bash[80002]: cluster 2023-10-09T06:31:51.069499+0000 mgr.y (mgr.25723) 809 : cluster 0 pgmap v809: 89 pgs: 89 active+clean; 2.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:31:52.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:52.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:52.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:52.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:52.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:52.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:52.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:52.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:52.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:52.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:53.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:53.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:53.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:53.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:53.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:53.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:53.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:53.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:53.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:53.571 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:53.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:53.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:53.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:53.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:53.834 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:53.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:53.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:53.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:53.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:54.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:54.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:54.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:54.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:54.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:54.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:54 smithi125 bash[77299]: cluster 2023-10-09T06:31:53.070031+0000 mgr.y (mgr.25723) 810 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:54.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:54 smithi125 bash[80002]: cluster 2023-10-09T06:31:53.070031+0000 mgr.y (mgr.25723) 810 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:54.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:54.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:54 smithi155 bash[59495]: cluster 2023-10-09T06:31:53.070031+0000 mgr.y (mgr.25723) 810 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:31:54.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:54.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:54.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:54.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:54.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:54.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:54.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:54.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:54.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:54.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:55.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:55.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:55.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:55.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:55.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:55.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:55.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:55.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:55.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:55.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:55.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:55.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:56.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:56.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:56.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:56.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:56.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:56.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:56.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:56 smithi125 bash[77299]: cluster 2023-10-09T06:31:55.070676+0000 mgr.y (mgr.25723) 811 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:31:56.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:56 smithi125 bash[80002]: cluster 2023-10-09T06:31:55.070676+0000 mgr.y (mgr.25723) 811 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:31:56.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:56 smithi155 bash[59495]: cluster 2023-10-09T06:31:55.070676+0000 mgr.y (mgr.25723) 811 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:31:56.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:56.444 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:56.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:56.580 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:56.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:56.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:56.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:56.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:56.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:56.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:56.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:56.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:57.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:57.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:57.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:57.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:57.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:57.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:57.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:57.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:57.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:58.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:58.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:58.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:58.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:58.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:58.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:58.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:58 smithi155 bash[59495]: cluster 2023-10-09T06:31:57.071216+0000 mgr.y (mgr.25723) 812 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:31:58.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:58.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:58.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:58.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:58 smithi125 bash[77299]: cluster 2023-10-09T06:31:57.071216+0000 mgr.y (mgr.25723) 812 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:31:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:58 smithi125 bash[80002]: cluster 2023-10-09T06:31:57.071216+0000 mgr.y (mgr.25723) 812 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:31:58.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:58.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:58.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:58.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:58.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:58.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:58.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:59.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:59.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:59.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:59.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:59.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:59.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:59.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:59.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:59.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:59.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:59.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:59.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:59.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:31:59.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:00.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:00.023 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:31:59 smithi155 bash[59495]: cluster 2023-10-09T06:31:59.071746+0000 mgr.y (mgr.25723) 813 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T06:32:00.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:00.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:31:59 smithi125 bash[80002]: cluster 2023-10-09T06:31:59.071746+0000 mgr.y (mgr.25723) 813 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T06:32:00.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:31:59 smithi125 bash[77299]: cluster 2023-10-09T06:31:59.071746+0000 mgr.y (mgr.25723) 813 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T06:32:00.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:00.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:00.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:00.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:00.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:00.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:00.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:00.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:00.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:00.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:00.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:00.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:00.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:00.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:00.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:01.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:01.238 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:01] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.33.4" 2023-10-09T06:32:01.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:01.295 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:32:01.349 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:32:01.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.424 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:32:01.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:01.463 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:32:01.463 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:32:01.463 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:32:01.463 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:32:01.463 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:32:01.489 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:32:01.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:32:01.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:01.589 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:32:01.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:01.638 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:32:01.680 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.695 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.696 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:32:01.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:01.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:01.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:01.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:01.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:01.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:02.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:02.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.134 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: cluster 2023-10-09T06:32:01.072306+0000 mgr.y (mgr.25723) 814 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:32:02.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: audit 2023-10-09T06:32:01.235780+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.125:0/857587078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: audit 2023-10-09T06:32:01.896233+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: audit 2023-10-09T06:32:01.899636+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.125:0/534058792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:02.153 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:32:02.156 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:32:02.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: cluster 2023-10-09T06:32:01.072306+0000 mgr.y (mgr.25723) 814 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: audit 2023-10-09T06:32:01.235780+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.125:0/857587078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: audit 2023-10-09T06:32:01.896233+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: audit 2023-10-09T06:32:01.899636+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.125:0/534058792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: audit 2023-10-09T06:32:01.969584+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[80002]: audit 2023-10-09T06:32:01.972834+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.125:0/3444544059' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: audit 2023-10-09T06:32:01.969584+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:02 smithi125 bash[77299]: audit 2023-10-09T06:32:01.972834+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.125:0/3444544059' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: cluster 2023-10-09T06:32:01.072306+0000 mgr.y (mgr.25723) 814 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:32:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: audit 2023-10-09T06:32:01.235780+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.125:0/857587078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: audit 2023-10-09T06:32:01.896233+0000 mon.a (mon.0) 1012 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: audit 2023-10-09T06:32:01.899636+0000 mon.b (mon.1) 19 : audit 1 from='client.? 172.21.15.125:0/534058792' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: audit 2023-10-09T06:32:01.969584+0000 mon.a (mon.0) 1013 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:02 smithi155 bash[59495]: audit 2023-10-09T06:32:01.972834+0000 mon.b (mon.1) 20 : audit 1 from='client.? 172.21.15.125:0/3444544059' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:32:02.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:02.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:02.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:02.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:02.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:02.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:02.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:02.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:02.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:02.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:02.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:03.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:03.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:03.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:03.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:03.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:03.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.126223+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.141998+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: audit 2023-10-09T06:32:02.142100+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: audit 2023-10-09T06:32:02.142337+0000 mon.a (mon.0) 1016 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:32:03.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.142587+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:32:03.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.152791+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.165109+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:03 smithi155 bash[59495]: cluster 2023-10-09T06:32:02.170945+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.485 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.126223+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.141998+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: audit 2023-10-09T06:32:02.142100+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: audit 2023-10-09T06:32:02.142337+0000 mon.a (mon.0) 1016 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.142587+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:32:03.497 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.152791+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.165109+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[77299]: cluster 2023-10-09T06:32:02.170945+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.126223+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.141998+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: audit 2023-10-09T06:32:02.142100+0000 mon.a (mon.0) 1015 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: audit 2023-10-09T06:32:02.142337+0000 mon.a (mon.0) 1016 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.142587+0000 mon.a (mon.0) 1017 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:32:03.498 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.152791+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.165109+0000 osd.4 (osd.4) 116 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:03 smithi125 bash[80002]: cluster 2023-10-09T06:32:02.170945+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e185 with expected crc 2023-10-09T06:32:03.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:03.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:03.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:03.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:03.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:03.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:03.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:03.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:03.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:04.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:04.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:04.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:04.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[77299]: cluster 2023-10-09T06:32:03.075242+0000 mgr.y (mgr.25723) 815 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:32:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[77299]: audit 2023-10-09T06:32:03.589772+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:04.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[77299]: audit 2023-10-09T06:32:03.591300+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:04.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[80002]: cluster 2023-10-09T06:32:03.075242+0000 mgr.y (mgr.25723) 815 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:32:04.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[80002]: audit 2023-10-09T06:32:03.589772+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:04.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:04 smithi125 bash[80002]: audit 2023-10-09T06:32:03.591300+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:04.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:04.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:04.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:04 smithi155 bash[59495]: cluster 2023-10-09T06:32:03.075242+0000 mgr.y (mgr.25723) 815 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-09T06:32:04.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:04 smithi155 bash[59495]: audit 2023-10-09T06:32:03.589772+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:04.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:04 smithi155 bash[59495]: audit 2023-10-09T06:32:03.591300+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:04.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:04.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:04.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:04.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:04.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:04.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:04.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:04.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:04.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:32:04.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:05.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:05.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:05.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:05.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:05.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:05.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:05.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:05.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:05.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:05.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:05.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:05.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:05.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:05.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:05.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:05.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:05.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:06.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:06.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:32:06.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:06.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:06.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:06.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:06 smithi155 bash[59495]: cluster 2023-10-09T06:32:05.075910+0000 mgr.y (mgr.25723) 816 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:32:06.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:06.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:06.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:06 smithi125 bash[77299]: cluster 2023-10-09T06:32:05.075910+0000 mgr.y (mgr.25723) 816 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:32:06.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:06 smithi125 bash[80002]: cluster 2023-10-09T06:32:05.075910+0000 mgr.y (mgr.25723) 816 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:32:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:06.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:06.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:06.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:06.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:06.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:06.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:06.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:06.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:06.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:07.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:07.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:07.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:07.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:07.323 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:07.354 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:07.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:07.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:07.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:07.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:07.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:07.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:07.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:07.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:08.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:32:08.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:08.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:32:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:08.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:08.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:08.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:08.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:08 smithi155 bash[59495]: cluster 2023-10-09T06:32:07.076357+0000 mgr.y (mgr.25723) 817 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T06:32:08.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:08 smithi125 bash[77299]: cluster 2023-10-09T06:32:07.076357+0000 mgr.y (mgr.25723) 817 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T06:32:08.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:08 smithi125 bash[80002]: cluster 2023-10-09T06:32:07.076357+0000 mgr.y (mgr.25723) 817 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-09T06:32:08.443 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:08.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:08.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:08.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:08.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:08.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:08.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:08.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:08.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:09.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:09.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:09.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:09.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:09 smithi125 bash[77299]: audit 2023-10-09T06:32:08.429459+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:32:09.224 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:09 smithi125 bash[80002]: audit 2023-10-09T06:32:08.429459+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:32:09.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:09.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:09.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:09.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:09 smithi155 bash[59495]: audit 2023-10-09T06:32:08.429459+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:32:09.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:09.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:09.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:09.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:09.770 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:09.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:09.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:09.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:09.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:09.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:10.030 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:10.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:10.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:10.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:10.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:10.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:10.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:10 smithi155 bash[59495]: cluster 2023-10-09T06:32:09.076945+0000 mgr.y (mgr.25723) 818 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:32:10.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:10.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:10.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:10.579 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:10 smithi125 bash[77299]: cluster 2023-10-09T06:32:09.076945+0000 mgr.y (mgr.25723) 818 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:32:10.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:10 smithi125 bash[80002]: cluster 2023-10-09T06:32:09.076945+0000 mgr.y (mgr.25723) 818 : cluster 0 pgmap v819: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-10-09T06:32:10.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:10.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:10.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:10.792 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:10.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:10.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:10.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:10.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:11.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:11.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:11.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:11.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:11.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:11.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:11.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:11.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:11.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:11.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:11.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:11] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.33.4" 2023-10-09T06:32:11.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:11.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:11.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:11.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:11.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:11.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:11.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:11.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:12.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:12.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:12.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:12.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:12.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:12.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:12 smithi125 bash[77299]: cluster 2023-10-09T06:32:11.077542+0000 mgr.y (mgr.25723) 819 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:32:12.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:12 smithi125 bash[80002]: cluster 2023-10-09T06:32:11.077542+0000 mgr.y (mgr.25723) 819 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:32:12.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:12.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:12.472 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:12 smithi155 bash[59495]: cluster 2023-10-09T06:32:11.077542+0000 mgr.y (mgr.25723) 819 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:32:12.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:12.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:12.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:12.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:12.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:12.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:12.835 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:12.840 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:12.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:12.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:12.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:12.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:12.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:13.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:13.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:13.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:13.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:13.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:13.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:13.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:13.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:13.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:13.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:13.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:13.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:13.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:14.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:14.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:14.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:14.162 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:14.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.287 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:14.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:14.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:14.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:14 smithi125 bash[77299]: cluster 2023-10-09T06:32:13.078044+0000 mgr.y (mgr.25723) 820 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:32:14.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:14 smithi125 bash[80002]: cluster 2023-10-09T06:32:13.078044+0000 mgr.y (mgr.25723) 820 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:32:14.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:14.526 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:14 smithi155 bash[59495]: cluster 2023-10-09T06:32:13.078044+0000 mgr.y (mgr.25723) 820 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:32:14.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:14.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:14.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:14.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:14.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:14.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:15.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:15.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:15.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:15.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:15.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:15.236 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[77299]: audit 2023-10-09T06:32:14.184037+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[77299]: audit 2023-10-09T06:32:14.194274+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[77299]: audit 2023-10-09T06:32:14.744166+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[77299]: audit 2023-10-09T06:32:14.755842+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:15.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[80002]: audit 2023-10-09T06:32:14.184037+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[80002]: audit 2023-10-09T06:32:14.194274+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[80002]: audit 2023-10-09T06:32:14.744166+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.238 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:15 smithi125 bash[80002]: audit 2023-10-09T06:32:14.755842+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:15.350 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:15 smithi155 bash[59495]: audit 2023-10-09T06:32:14.184037+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.351 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:15 smithi155 bash[59495]: audit 2023-10-09T06:32:14.194274+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.351 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:15 smithi155 bash[59495]: audit 2023-10-09T06:32:14.744166+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.351 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:15 smithi155 bash[59495]: audit 2023-10-09T06:32:14.755842+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:15.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:15.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:15.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:15.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:15.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:15.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:15.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:15.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:15.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:15.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:15.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:15.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:16.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:16.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:16.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:16.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:16.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[77299]: cluster 2023-10-09T06:32:15.078831+0000 mgr.y (mgr.25723) 821 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:32:16.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[77299]: audit 2023-10-09T06:32:15.335201+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:16.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[77299]: audit 2023-10-09T06:32:15.336548+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:16.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[77299]: audit 2023-10-09T06:32:15.343325+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:16.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[80002]: cluster 2023-10-09T06:32:15.078831+0000 mgr.y (mgr.25723) 821 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:32:16.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[80002]: audit 2023-10-09T06:32:15.335201+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:16.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[80002]: audit 2023-10-09T06:32:15.336548+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:16.278 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:16 smithi125 bash[80002]: audit 2023-10-09T06:32:15.343325+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:16.299 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:16 smithi155 bash[59495]: cluster 2023-10-09T06:32:15.078831+0000 mgr.y (mgr.25723) 821 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:32:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:16 smithi155 bash[59495]: audit 2023-10-09T06:32:15.335201+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:16 smithi155 bash[59495]: audit 2023-10-09T06:32:15.336548+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:16 smithi155 bash[59495]: audit 2023-10-09T06:32:15.343325+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:32:16.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:16.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:16.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:16.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:16.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:16.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:16.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:16.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:16.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:16.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:16.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:16.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:17.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:17.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:17.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:17.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:17.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:17.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:17.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:17.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:17.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:17.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:17.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:17.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:17.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:17.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:17.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:18.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:18.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:18.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:18.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:18.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:18.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:18.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:18.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:18.442 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:18 smithi155 bash[59495]: cluster 2023-10-09T06:32:17.079359+0000 mgr.y (mgr.25723) 822 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:32:18.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:18.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:18.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:18.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:18 smithi125 bash[77299]: cluster 2023-10-09T06:32:17.079359+0000 mgr.y (mgr.25723) 822 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:32:18.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:18 smithi125 bash[80002]: cluster 2023-10-09T06:32:17.079359+0000 mgr.y (mgr.25723) 822 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T06:32:18.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:18.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:18.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:18.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:18.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:18.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:18.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:19.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:19.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:19.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:19.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:19 smithi155 bash[59495]: audit 2023-10-09T06:32:18.586879+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:19.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:19.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:19 smithi125 bash[77299]: audit 2023-10-09T06:32:18.586879+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:19.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:19 smithi125 bash[80002]: audit 2023-10-09T06:32:18.586879+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:19.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:19.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:19.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:19.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:19.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:19.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:19.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:20.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:20.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:20.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:20.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:20 smithi125 bash[77299]: cluster 2023-10-09T06:32:19.080105+0000 mgr.y (mgr.25723) 823 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:32:20.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:20 smithi125 bash[80002]: cluster 2023-10-09T06:32:19.080105+0000 mgr.y (mgr.25723) 823 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:32:20.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:20.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:20.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:20.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:20.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:20.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:20.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:20 smithi155 bash[59495]: cluster 2023-10-09T06:32:19.080105+0000 mgr.y (mgr.25723) 823 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:32:20.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:20.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:20.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:20.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:20.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:21.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.145 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:21.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:21.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:21] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:32:21.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:21.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:21.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:21.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:21.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:21.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:21.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:21.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:21.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:21.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:21.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:22.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:22.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:22.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:22.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:22.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:22.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:22 smithi125 bash[80002]: cluster 2023-10-09T06:32:21.080674+0000 mgr.y (mgr.25723) 824 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 976 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T06:32:22.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:22 smithi125 bash[77299]: cluster 2023-10-09T06:32:21.080674+0000 mgr.y (mgr.25723) 824 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 976 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T06:32:22.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:22.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:22.451 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:22 smithi155 bash[59495]: cluster 2023-10-09T06:32:21.080674+0000 mgr.y (mgr.25723) 824 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 976 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T06:32:22.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:22.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:22.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:22.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:22.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:22.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:22.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:22.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:22.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:22.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:22.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:23.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:23.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:23.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:23.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:23.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:23.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:23.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:23.402 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:23.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:23.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:23.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:23.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:23.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:23.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:23.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:23.861 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:32:23.861 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.44939.0:37209 6.b e146/142 6:da7cb9eb:::benchmark_data_smithi125_130224_object37208:head v 146'2366, mlcod=146'2364)", 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:42.294578+0000", 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 521.548776299, 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.170191336, 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.862 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.294578+0000", 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.294578+0000", 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.863 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.294577+0000", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.294607+0000", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0378999999999998e-05 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.294620+0000", 2023-10-09T06:32:23.864 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2862e-05 2023-10-09T06:32:23.865 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.865 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.865 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.865 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.452713+0000", 2023-10-09T06:32:23.865 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15809271 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.452774+0000", 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0853999999999997e-05 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.866 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.452800+0000", 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6604000000000001e-05 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.464732+0000", 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011931310000000001 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:42.464769+0000", 2023-10-09T06:32:23.867 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7474000000000001e-05 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:39525 6.4 6:21de881d:::benchmark_data_smithi125_130224_object39524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:23:43.970826+0000", 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 519.87252763699996, 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20037201900000001, 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.868 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 39525 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970826+0000", 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.869 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970826+0000", 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970825+0000", 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.870 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970846+0000", 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0752000000000001e-05 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970857+0000", 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0698e-05 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.871 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970862+0000", 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.575e-06 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.970906+0000", 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4125999999999997e-05 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.971037+0000", 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000130319 2023-10-09T06:32:23.872 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.971167+0000", 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013070399999999999 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.972576+0000", 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001408912 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.873 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:43.972930+0000", 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00035390399999999999 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.171088+0000", 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19815784 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.874 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.171149+0000", 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0705000000000002e-05 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:23:44.171198+0000", 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9660000000000002e-05 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.875 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:117536 6.4 6:233b0323:::benchmark_data_smithi125_130224_object6345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:25:01.730820+0000", 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 442.11253378700002, 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20515823899999999, 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 117536 2023-10-09T06:32:23.876 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730820+0000", 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730820+0000", 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.877 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730818+0000", 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730828+0000", 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.878 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730848+0000", 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9338999999999999e-05 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730859+0000", 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1323e-05 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.879 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.730982+0000", 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000122748 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.731139+0000", 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015752999999999999 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.731252+0000", 2023-10-09T06:32:23.880 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011317800000000001 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.735770+0000", 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045174380000000004 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.738680+0000", 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0029106840000000002 2023-10-09T06:32:23.881 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.935563+0000", 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19688265999999999 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.935862+0000", 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029864799999999999 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.882 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.935979+0000", 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116848 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.44939.0:117550 6.4 6:2ad34f55:::benchmark_data_smithi125_130224_object6359:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:25:01.753774+0000", 2023-10-09T06:32:23.883 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 442.08957980000002, 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19597293800000001, 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.44939", 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/3174106459", 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 117550 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.884 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753774+0000", 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753774+0000", 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.885 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753773+0000", 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753780+0000", 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8709999999999999e-06 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.886 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753794+0000", 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3987e-05 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753802+0000", 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4319999999999999e-06 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.887 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753855+0000", 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3127000000000001e-05 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.753978+0000", 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012313600000000001 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.754089+0000", 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110628 2023-10-09T06:32:23.888 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.758388+0000", 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004298975 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.758628+0000", 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024001299999999999 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.889 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.949667+0000", 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.191039077 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.949712+0000", 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4771999999999998e-05 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.890 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:25:01.949747+0000", 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5540000000000002e-05 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:16619 7.4 e173/165 7:232f6f1f:::benchmark_data_smithi125_153312_object16618:head v 173'1019, mlcod=173'1016)", 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.211367+0000", 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.63198708499999, 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21057367299999999, 2023-10-09T06:32:23.891 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211367+0000", 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.892 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211367+0000", 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211366+0000", 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211396+0000", 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9779000000000001e-05 2023-10-09T06:32:23.893 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211406+0000", 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3919999999999992e-06 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211412+0000", 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3040000000000001e-06 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.894 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212595+0000", 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011830359999999999 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212617+0000", 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2061000000000001e-05 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.895 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.421865+0000", 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20924780400000001 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.421941+0000", 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5839000000000004e-05 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.896 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:16626 7.d 7:bfac94f7:::benchmark_data_smithi125_153312_object16625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.211523+0000", 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.63183149, 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21329588599999999, 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.897 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 16626 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211523+0000", 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211523+0000", 2023-10-09T06:32:23.898 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211521+0000", 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211578+0000", 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6847e-05 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.899 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211590+0000", 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2344000000000001e-05 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211598+0000", 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.900 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211634+0000", 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6297000000000002e-05 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211726+0000", 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.2422000000000004e-05 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.901 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211806+0000", 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9308999999999996e-05 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420795+0000", 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20898921600000001 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423844+0000", 2023-10-09T06:32:23.902 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0030487370000000001 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.424734+0000", 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00089080400000000003 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.424780+0000", 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5473999999999999e-05 2023-10-09T06:32:23.903 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.424819+0000", 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8634999999999999e-05 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:16627 7.b 7:d844efd9:::benchmark_data_smithi125_153312_object16626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:32:23.904 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.211610+0000", 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.631744529, 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21163530799999999, 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 16627 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.905 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211610+0000", 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211610+0000", 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.906 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211609+0000", 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211628+0000", 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8981999999999999e-05 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211635+0000", 2023-10-09T06:32:23.907 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5749999999999997e-06 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211640+0000", 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.5310000000000001e-06 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211784+0000", 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014413499999999999 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.908 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.211999+0000", 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000214272 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212128+0000", 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012949200000000001 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.909 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420584+0000", 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20845644799999999 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420683+0000", 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8938999999999996e-05 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.910 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423161+0000", 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0024775779999999998 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423209+0000", 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7719000000000002e-05 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423245+0000", 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6164999999999997e-05 2023-10-09T06:32:23.911 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:16622 7.4 e173/165 7:27893510:::benchmark_data_smithi125_153312_object16621:head v 173'1020, mlcod=173'1016)", 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.212644+0000", 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.630709833, 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21075091900000001, 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.912 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212644+0000", 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212644+0000", 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.913 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212643+0000", 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212670+0000", 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6248999999999999e-05 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.914 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212677+0000", 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0759999999999997e-06 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.212681+0000", 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5340000000000001e-06 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.915 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213272+0000", 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00059077299999999995 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213289+0000", 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7422999999999999e-05 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423331+0000", 2023-10-09T06:32:23.916 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.21004173900000001 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423395+0000", 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4035999999999995e-05 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.917 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:16629 7.9 e173/165 7:9fd68fa1:::benchmark_data_smithi125_153312_object16628:head v 173'1044, mlcod=173'1042)", 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.213575+0000", 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.62977906500001, 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.209757952, 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213575+0000", 2023-10-09T06:32:23.918 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213575+0000", 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213574+0000", 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.213599+0000", 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5154999999999999e-05 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420557+0000", 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20695725400000001 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.920 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420574+0000", 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7795000000000002e-05 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420633+0000", 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8996999999999998e-05 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.921 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420664+0000", 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0051000000000001e-05 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423300+0000", 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026366950000000001 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.423333+0000", 2023-10-09T06:32:23.922 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2799000000000002e-05 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.45170.0:16627 7.b e173/169 ondisk, result = 0)", 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:45.214509+0000", 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 218.62884498700001, 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.206178691, 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:32:23.923 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.214509+0000", 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.214509+0000", 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.924 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.214508+0000", 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.214516+0000", 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9470000000000003e-06 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.925 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420561+0000", 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20604470899999999 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420595+0000", 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4362000000000001e-05 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.926 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420665+0000", 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9596000000000002e-05 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420676+0000", 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1433999999999999e-05 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:45.420688+0000", 2023-10-09T06:32:23.927 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1327e-05 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:85661 7.b 7:d92892e7:::benchmark_data_smithi125_153312_object85660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:43.561258+0000", 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 160.28209599100001, 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.221419849, 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.928 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 85661 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561258+0000", 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.929 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561258+0000", 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561256+0000", 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.930 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561283+0000", 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6415999999999999e-05 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561303+0000", 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9809999999999998e-05 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.931 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561312+0000", 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3899999999999999e-06 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561499+0000", 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018658100000000001 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561631+0000", 2023-10-09T06:32:23.932 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013245600000000001 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561764+0000", 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013305899999999999 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574438+0000", 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012673528 2023-10-09T06:32:23.933 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574558+0000", 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011985399999999999 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782565+0000", 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20800786199999999 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.934 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782628+0000", 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2620999999999996e-05 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782678+0000", 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9969999999999998e-05 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.935 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:85663 7.b 7:ddc4b9fb:::benchmark_data_smithi125_153312_object85662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:43.561324+0000", 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 160.28203030200001, 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22149016599999999, 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.936 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 85663 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561324+0000", 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561324+0000", 2023-10-09T06:32:23.937 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561323+0000", 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561342+0000", 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8983000000000001e-05 2023-10-09T06:32:23.938 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561349+0000", 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.6019999999999996e-06 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561354+0000", 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3260000000000003e-06 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.939 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.564702+0000", 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033478900000000001 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.564803+0000", 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010063199999999999 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:32:23.940 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.564900+0000", 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7350999999999997e-05 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574461+0000", 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0095608910000000002 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574579+0000", 2023-10-09T06:32:23.941 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011805 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782727+0000", 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20814848 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782774+0000", 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6665000000000001e-05 2023-10-09T06:32:23.942 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782814+0000", 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9827999999999999e-05 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:85664 7.b 7:dc4afc5a:::benchmark_data_smithi125_153312_object85663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:43.561359+0000", 2023-10-09T06:32:23.943 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 160.281995467, 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22158040300000001, 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 85664 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.944 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561359+0000", 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561359+0000", 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.945 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561358+0000", 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561376+0000", 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7589e-05 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561382+0000", 2023-10-09T06:32:23.946 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.857e-06 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.561387+0000", 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7960000000000003e-06 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.568772+0000", 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0073856909999999998 2023-10-09T06:32:23.947 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.568932+0000", 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016023400000000001 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.569083+0000", 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015086799999999999 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.948 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574477+0000", 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0053932329999999999 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574592+0000", 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011568799999999999 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.949 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782855+0000", 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20826296599999999 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782901+0000", 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5429000000000002e-05 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.782939+0000", 2023-10-09T06:32:23.950 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8436999999999998e-05 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:85675 7.b 7:daa8f3a2:::benchmark_data_smithi125_153312_object85674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:43.574823+0000", 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 160.26853136400001, 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.20896236100000001, 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.951 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 85675 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574823+0000", 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.952 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574823+0000", 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574821+0000", 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.953 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574844+0000", 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2778000000000001e-05 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574859+0000", 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5067e-05 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.954 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574867+0000", 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9480000000000008e-06 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.574904+0000", 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7160999999999999e-05 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.575011+0000", 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106327 2023-10-09T06:32:23.955 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.956 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.957 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:32:23.957 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.575141+0000", 2023-10-09T06:32:23.957 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013076900000000001 2023-10-09T06:32:23.957 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.957 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.577549+0000", 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002408187 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.578449+0000", 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00089908999999999996 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.958 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.783569+0000", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.205120583 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.783744+0000", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017479600000000001 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:43.783785+0000", 2023-10-09T06:32:23.959 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1143000000000002e-05 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:155386 7.d 7:bb6685c6:::benchmark_data_smithi125_153312_object50699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:18.848077+0000", 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 64.995277514999998, 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.169564455, 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.960 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 155386 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848077+0000", 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848077+0000", 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848075+0000", 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.962 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848082+0000", 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848095+0000", 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3044000000000001e-05 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848102+0000", 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:32:23.963 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848672+0000", 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00056963599999999999 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848777+0000", 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000104578 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.964 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.848882+0000", 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105083 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.853923+0000", 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050410050000000003 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.965 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:18.860015+0000", 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0060925930000000003 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:19.017550+0000", 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.15753499900000001 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:19.017601+0000", 2023-10-09T06:32:23.966 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0818e-05 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:19.017641+0000", 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9938999999999998e-05 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.967 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:162606 7.d 7:ba3789d8:::benchmark_data_smithi125_153312_object57919:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:29.732613+0000", 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.110740684, 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22315761200000001, 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 162606 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.968 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732613+0000", 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732613+0000", 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.969 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732612+0000", 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732618+0000", 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3099999999999997e-06 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.970 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732631+0000", 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2531000000000001e-05 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.732650+0000", 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9174999999999999e-05 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.734586+0000", 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001936152 2023-10-09T06:32:23.971 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.734706+0000", 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000120191 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.734813+0000", 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010712799999999999 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.972 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.744174+0000", 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0093604239999999991 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.757274+0000", 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013099883 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.973 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955681+0000", 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.19840687100000001 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955732+0000", 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0886e-05 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.974 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955771+0000", 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9507e-05 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:162618 7.d 7:b8e09a4a:::benchmark_data_smithi125_153312_object57931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:29.745193+0000", 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.098160698999997, 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.210737959, 2023-10-09T06:32:23.975 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 162618 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745193+0000", 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.976 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745193+0000", 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745192+0000", 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.977 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745199+0000", 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9229999999999996e-06 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745212+0000", 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2673e-05 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.978 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745221+0000", 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8249999999999994e-06 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745270+0000", 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9898999999999998e-05 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745393+0000", 2023-10-09T06:32:23.979 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012262500000000001 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.745514+0000", 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121289 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753227+0000", 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077130039999999999 2023-10-09T06:32:23.980 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.757308+0000", 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040808789999999999 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955826+0000", 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.198517482 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.981 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955899+0000", 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3489000000000001e-05 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.955931+0000", 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2131000000000001e-05 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.982 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:162621 7.d 7:b3c16d5a:::benchmark_data_smithi125_153312_object57934:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:29.753872+0000", 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.089482547000003, 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.23254035300000001, 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.983 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 162621 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753872+0000", 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753872+0000", 2023-10-09T06:32:23.984 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753870+0000", 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753877+0000", 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7920000000000004e-06 2023-10-09T06:32:23.985 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753889+0000", 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2607e-05 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753897+0000", 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8650000000000001e-06 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.986 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.753932+0000", 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4854999999999997e-05 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.754016+0000", 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4185999999999997e-05 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.754098+0000", 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1417999999999999e-05 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.757356+0000", 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0032582850000000001 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758005+0000", 2023-10-09T06:32:23.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00064920899999999998 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986319+0000", 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22831342399999999 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986370+0000", 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1341000000000002e-05 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986412+0000", 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1959999999999998e-05 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.991 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:162622 7.d 7:bd6568c7:::benchmark_data_smithi125_153312_object57935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:32:23.991 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:31:29.758950+0000", 2023-10-09T06:32:23.991 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 54.084404472999999, 2023-10-09T06:32:23.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.227651675, 2023-10-09T06:32:23.991 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:23.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:23.992 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:23.992 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 162622 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758950+0000", 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758950+0000", 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758949+0000", 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:23.994 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758954+0000", 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9910000000000002e-06 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758963+0000", 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1269999999999992e-06 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.758968+0000", 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6280000000000002e-06 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.759002+0000", 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3636000000000001e-05 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.759086+0000", 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.4116000000000006e-05 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.759161+0000", 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5202000000000003e-05 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.761867+0000", 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027053709999999998 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.762311+0000", 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000444096 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986454+0000", 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.224143493 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986562+0000", 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000107557 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:23.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:31:29.986601+0000", 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9499000000000002e-05 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:195349 7.4 e185/165 7:21217dec:::benchmark_data_smithi125_153312_object90662:head v 185'12098, mlcod=185'12096)", 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:32:20.090599+0000", 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 3.7527552589999997, 2023-10-09T06:32:24.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.17699899299999999, 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.090599+0000", 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:32:24.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.090599+0000", 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.090598+0000", 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.090609+0000", 2023-10-09T06:32:24.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0788000000000001e-05 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.212702+0000", 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.12209291899999999 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.212716+0000", 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4859e-05 2023-10-09T06:32:24.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.212762+0000", 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5469999999999997e-05 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.212786+0000", 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4133000000000001e-05 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.267563+0000", 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.054776850000000002 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:20.267598+0000", 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4907999999999998e-05 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:24.005 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:32:24.006 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:32:24.006 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:32:24.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:24.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:24.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:24.273 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:24.282 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:24 smithi125 bash[77299]: cluster 2023-10-09T06:32:23.081207+0000 mgr.y (mgr.25723) 825 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 872 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:24.282 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:24 smithi125 bash[80002]: cluster 2023-10-09T06:32:23.081207+0000 mgr.y (mgr.25723) 825 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 872 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:24.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:24.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.420 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:24 smithi155 bash[59495]: cluster 2023-10-09T06:32:23.081207+0000 mgr.y (mgr.25723) 825 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 872 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:24.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:24.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:24.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:24.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:24.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:24.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:24.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:24.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:24.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:24.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:25.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:25.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:25.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:25.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:25.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:25.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:25.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:25.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:25.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:25.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:25.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:25.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:25.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:25.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:25.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:25.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:26.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:26.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:26.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:26.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:26.276 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:26 smithi125 bash[77299]: cluster 2023-10-09T06:32:25.081824+0000 mgr.y (mgr.25723) 826 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 769 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:26.276 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:26 smithi125 bash[80002]: cluster 2023-10-09T06:32:25.081824+0000 mgr.y (mgr.25723) 826 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 769 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:26.302 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:26 smithi155 bash[59495]: cluster 2023-10-09T06:32:25.081824+0000 mgr.y (mgr.25723) 826 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 769 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:32:26.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:26.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:26.549 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:26.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:26.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:26.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:26.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:26.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:26.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:26.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:26.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:26.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:26.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:26.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:27.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:27.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:27.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:27.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:27.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:27.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:27.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:27.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:27.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:27.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:27.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:27.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:27.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:27.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:27.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:28.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:28.057 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:28.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:28.160 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:28.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:28.215 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:28 smithi155 bash[59495]: cluster 2023-10-09T06:32:27.082539+0000 mgr.y (mgr.25723) 827 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 698 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:32:28.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:28.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:28.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:28 smithi125 bash[77299]: cluster 2023-10-09T06:32:27.082539+0000 mgr.y (mgr.25723) 827 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 698 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:32:28.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:28 smithi125 bash[80002]: cluster 2023-10-09T06:32:27.082539+0000 mgr.y (mgr.25723) 827 : cluster 0 pgmap v828: 89 pgs: 89 active+clean; 698 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:32:28.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:28.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:28.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:28.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:28.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:28.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:28.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:28.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:28.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:28.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:28.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:28.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:29.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:29.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:32:29.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:29.233 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:32:29.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:29.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:29.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:29.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:29.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:29 smithi155 bash[59495]: audit 2023-10-09T06:32:28.392980+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:32:29.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:29.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:29 smithi125 bash[77299]: audit 2023-10-09T06:32:28.392980+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:32:29.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:29 smithi125 bash[80002]: audit 2023-10-09T06:32:28.392980+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:32:29.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:29.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:29.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:29.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:29.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:29.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:29.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:30.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:30.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:30.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:30.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:30.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:30.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:30.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:32:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: cluster 2023-10-09T06:32:29.083127+0000 mgr.y (mgr.25723) 828 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 625 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:32:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: audit 2023-10-09T06:32:29.219128+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:32:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: cluster 2023-10-09T06:32:29.219209+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:32:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: cluster 2023-10-09T06:32:29.225462+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: cluster 2023-10-09T06:32:29.245383+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:30 smithi155 bash[59495]: cluster 2023-10-09T06:32:29.252074+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.407 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:32:30.451 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:32:30.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: cluster 2023-10-09T06:32:29.083127+0000 mgr.y (mgr.25723) 828 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 625 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: audit 2023-10-09T06:32:29.219128+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: cluster 2023-10-09T06:32:29.219209+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: cluster 2023-10-09T06:32:29.225462+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: cluster 2023-10-09T06:32:29.245383+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[77299]: cluster 2023-10-09T06:32:29.252074+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: cluster 2023-10-09T06:32:29.083127+0000 mgr.y (mgr.25723) 828 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 625 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:32:30.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: audit 2023-10-09T06:32:29.219128+0000 mon.a (mon.0) 1030 : audit 1 from='client.? 172.21.15.125:0/4117684138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:32:30.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: cluster 2023-10-09T06:32:29.219209+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:32:30.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: cluster 2023-10-09T06:32:29.225462+0000 osd.4 (osd.4) 117 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: cluster 2023-10-09T06:32:29.245383+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:30 smithi125 bash[80002]: cluster 2023-10-09T06:32:29.252074+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e186 with expected crc 2023-10-09T06:32:30.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:30.495 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:32:30.536 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:32:30.572 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:32:30.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:32:30.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:30.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:30.645 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.653 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.654 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:32:30.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:30.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:30.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:30.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:30.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:30.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:31.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:31.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:31.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:31.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:31.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.287 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:31 smithi155 bash[59495]: audit 2023-10-09T06:32:30.308136+0000 mon.a (mon.0) 1032 : audit 0 from='client.? 172.21.15.125:0/1984885477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:31.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:31.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:31.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:31.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:31 smithi125 bash[77299]: audit 2023-10-09T06:32:30.308136+0000 mon.a (mon.0) 1032 : audit 0 from='client.? 172.21.15.125:0/1984885477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:31.554 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:31] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:32:31.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:31 smithi125 bash[80002]: audit 2023-10-09T06:32:30.308136+0000 mon.a (mon.0) 1032 : audit 0 from='client.? 172.21.15.125:0/1984885477' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:31.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:31.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:31.700 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:31.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:31.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:31.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:31.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:31.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:31.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:32:31.953 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:32.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:32.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:32.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:32.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:32.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:32.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:32.287 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:32:32.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:32.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:32.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:32.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[77299]: cluster 2023-10-09T06:32:31.083725+0000 mgr.y (mgr.25723) 829 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 578 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T06:32:32.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[77299]: audit 2023-10-09T06:32:32.172723+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.125:0/2496894330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[77299]: audit 2023-10-09T06:32:32.173158+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[80002]: cluster 2023-10-09T06:32:31.083725+0000 mgr.y (mgr.25723) 829 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 578 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T06:32:32.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[80002]: audit 2023-10-09T06:32:32.172723+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.125:0/2496894330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.536 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:32 smithi125 bash[80002]: audit 2023-10-09T06:32:32.173158+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:32 smithi155 bash[59495]: cluster 2023-10-09T06:32:31.083725+0000 mgr.y (mgr.25723) 829 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 578 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T06:32:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:32 smithi155 bash[59495]: audit 2023-10-09T06:32:32.172723+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.125:0/2496894330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:32 smithi155 bash[59495]: audit 2023-10-09T06:32:32.173158+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:32.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:32.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:32.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:32.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:32.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:32.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:32.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:32.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:33.067 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.100 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:33.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:33.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:33.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:33.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:33.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:33.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:33.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.252750+0000 mon.a (mon.0) 1034 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:33.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: audit 2023-10-09T06:32:32.267805+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:33.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.268124+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:32:33.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.274755+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.275380+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.277477+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.282094+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[77299]: cluster 2023-10-09T06:32:32.284746+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.252750+0000 mon.a (mon.0) 1034 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: audit 2023-10-09T06:32:32.267805+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.268124+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.274755+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.480 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.275380+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.277477+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.282094+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.481 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:33 smithi125 bash[80002]: cluster 2023-10-09T06:32:32.284746+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:33.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:33.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.252750+0000 mon.a (mon.0) 1034 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:33.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: audit 2023-10-09T06:32:32.267805+0000 mon.a (mon.0) 1035 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.268124+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.274755+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.275380+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.277477+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.282094+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:33 smithi155 bash[59495]: cluster 2023-10-09T06:32:32.284746+0000 osd.4 (osd.4) 118 : cluster 3 failed to encode map e187 with expected crc 2023-10-09T06:32:33.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:33.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:33.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:33.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:33.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:33.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:33.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:34.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:34.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:34.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:34.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:34.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:34.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:34.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:34.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:34.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:34.558 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:34 smithi125 bash[77299]: cluster 2023-10-09T06:32:33.084267+0000 mgr.y (mgr.25723) 830 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 481 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:32:34.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:34 smithi125 bash[77299]: audit 2023-10-09T06:32:33.585953+0000 mon.a (mon.0) 1037 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:34.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:34 smithi125 bash[80002]: cluster 2023-10-09T06:32:33.084267+0000 mgr.y (mgr.25723) 830 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 481 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:32:34.559 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:34 smithi125 bash[80002]: audit 2023-10-09T06:32:33.585953+0000 mon.a (mon.0) 1037 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:34.584 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:34 smithi155 bash[59495]: cluster 2023-10-09T06:32:33.084267+0000 mgr.y (mgr.25723) 830 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 481 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:32:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:34 smithi155 bash[59495]: audit 2023-10-09T06:32:33.585953+0000 mon.a (mon.0) 1037 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:34.691 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:34.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:34.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:34.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:34.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:34.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:34.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:34.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:34.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:34.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:35.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:35.114 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:35.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:32:35.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:35.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:35.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:35.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:35.408 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:35 smithi125 bash[77299]: cluster 2023-10-09T06:32:35.084858+0000 mgr.y (mgr.25723) 831 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 376 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:32:35.408 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:35 smithi125 bash[80002]: cluster 2023-10-09T06:32:35.084858+0000 mgr.y (mgr.25723) 831 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 376 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:32:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:35.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:35.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:35 smithi155 bash[59495]: cluster 2023-10-09T06:32:35.084858+0000 mgr.y (mgr.25723) 831 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 376 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:32:35.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:35.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:35.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:35.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:35.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:35.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:35.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:35.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:36.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:32:36.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:36.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:36.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:36.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:36.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:36.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:36.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:36.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:36.658 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:36.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.726 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:36.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:36.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:36.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:36.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:37.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:37.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:37.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:37.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:37.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:37.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:37.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:37.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:37.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:37.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:37.679 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:37.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:37.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:37.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:37.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:37.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:37.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:38.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:38.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:38.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:38.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:38.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:38.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:38.356 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:38 smithi125 bash[77299]: cluster 2023-10-09T06:32:37.085368+0000 mgr.y (mgr.25723) 832 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 298 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:38.356 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:38 smithi125 bash[80002]: cluster 2023-10-09T06:32:37.085368+0000 mgr.y (mgr.25723) 832 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 298 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:38.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:38.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.513 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:38 smithi155 bash[59495]: cluster 2023-10-09T06:32:37.085368+0000 mgr.y (mgr.25723) 832 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 298 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:38.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:38.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:38.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:38.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:38.807 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:38.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:38.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:39.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:39.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:39.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:39.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:39.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:39.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:39.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:39.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:39.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:39.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:39.612 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:39.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:39.709 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:39.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:39.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:39.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:39.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:39.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:40.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:40.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:40.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:39 smithi125 bash[77299]: cluster 2023-10-09T06:32:39.085947+0000 mgr.y (mgr.25723) 833 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 220 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:32:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:39 smithi125 bash[80002]: cluster 2023-10-09T06:32:39.085947+0000 mgr.y (mgr.25723) 833 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 220 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:32:40.141 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:39 smithi155 bash[59495]: cluster 2023-10-09T06:32:39.085947+0000 mgr.y (mgr.25723) 833 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 220 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:32:40.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:40.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:40.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:40.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:40.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:40.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:40.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:40.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:40.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:40.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:40.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:40.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:40.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:41.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:41.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:41.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:41.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:41.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:41.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:41] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-10-09T06:32:41.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.774 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 104684 objects 2023-10-09T06:32:41.775 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :154.591 2023-10-09T06:32:41.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:41.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:41.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:41.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:42.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:42.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:42.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:42.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:42.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:42.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:42.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:42 smithi125 bash[77299]: cluster 2023-10-09T06:32:41.088007+0000 mgr.y (mgr.25723) 834 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:32:42.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:42 smithi125 bash[80002]: cluster 2023-10-09T06:32:41.088007+0000 mgr.y (mgr.25723) 834 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:32:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:42.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:42.506 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:42 smithi155 bash[59495]: cluster 2023-10-09T06:32:41.088007+0000 mgr.y (mgr.25723) 834 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 168 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T06:32:42.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:42.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:42.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:42.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:42.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:42.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:42.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:42.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:42.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:43.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:43.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:43.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:43.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:43.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:43.597 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:43.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:43.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:43.810 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:43.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:43.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:43.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:44.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:44.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:44.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:44.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:44.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:44.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:44 smithi125 bash[80002]: cluster 2023-10-09T06:32:43.088826+0000 mgr.y (mgr.25723) 835 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 62 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:44.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:44 smithi125 bash[77299]: cluster 2023-10-09T06:32:43.088826+0000 mgr.y (mgr.25723) 835 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 62 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:44.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:44.483 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:44.496 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:44 smithi155 bash[59495]: cluster 2023-10-09T06:32:43.088826+0000 mgr.y (mgr.25723) 835 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 62 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:32:44.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:44.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:44.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:44.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:44.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:44.783 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:44.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:44.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:44.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:44.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:44.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:45.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:45.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:45.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:45.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:45.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:45.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:45.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:45.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:45.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:45.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:45.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:45.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:45.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:45.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:45.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:45.934 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:45.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:45.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:46.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:46.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:46.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:46.218 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:46 smithi155 bash[59495]: cluster 2023-10-09T06:32:45.089791+0000 mgr.y (mgr.25723) 836 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 10 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:32:46.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:46.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:46.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:46.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:46.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:46.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:46.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:46.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:46 smithi125 bash[77299]: cluster 2023-10-09T06:32:45.089791+0000 mgr.y (mgr.25723) 836 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 10 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:32:46.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:46 smithi125 bash[80002]: cluster 2023-10-09T06:32:45.089791+0000 mgr.y (mgr.25723) 836 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 10 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-09T06:32:46.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:46.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:46.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:46.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:46.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:46.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:46.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:46.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:46.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:47.018 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:47.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:47.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:47.269 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T06:32:47.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T06:32:47.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:47.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:47.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:47.439 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:47.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:47.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:47.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:47.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:47.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:47.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:47.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:47.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:48.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:48.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:48.173 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:32:48.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:48.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:48 smithi125 bash[77299]: cluster 2023-10-09T06:32:47.090476+0000 mgr.y (mgr.25723) 837 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s 2023-10-09T06:32:48.214 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:48 smithi125 bash[77299]: audit 2023-10-09T06:32:47.647129+0000 mon.a (mon.0) 1038 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:32:48.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:48 smithi125 bash[80002]: cluster 2023-10-09T06:32:47.090476+0000 mgr.y (mgr.25723) 837 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s 2023-10-09T06:32:48.214 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:48 smithi125 bash[80002]: audit 2023-10-09T06:32:47.647129+0000 mon.a (mon.0) 1038 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:32:48.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:48.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:48.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:48.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:48 smithi155 bash[59495]: cluster 2023-10-09T06:32:47.090476+0000 mgr.y (mgr.25723) 837 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s 2023-10-09T06:32:48.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:48 smithi155 bash[59495]: audit 2023-10-09T06:32:47.647129+0000 mon.a (mon.0) 1038 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:32:48.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:48.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:48.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:48.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:48.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:48.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:48.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:48.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:48.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:48.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:49.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:49.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:49.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:49.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:49.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[80002]: audit 2023-10-09T06:32:48.162979+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[80002]: cluster 2023-10-09T06:32:48.163076+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[80002]: cluster 2023-10-09T06:32:48.164437+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[80002]: cluster 2023-10-09T06:32:48.274500+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[80002]: audit 2023-10-09T06:32:48.587202+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[77299]: audit 2023-10-09T06:32:48.162979+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[77299]: cluster 2023-10-09T06:32:48.163076+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:32:49.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[77299]: cluster 2023-10-09T06:32:48.164437+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[77299]: cluster 2023-10-09T06:32:48.274500+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:49 smithi125 bash[77299]: audit 2023-10-09T06:32:48.587202+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:49 smithi155 bash[59495]: audit 2023-10-09T06:32:48.162979+0000 mon.a (mon.0) 1039 : audit 1 from='client.? 172.21.15.125:0/2185372410' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:32:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:49 smithi155 bash[59495]: cluster 2023-10-09T06:32:48.163076+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:32:49.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:49 smithi155 bash[59495]: cluster 2023-10-09T06:32:48.164437+0000 osd.4 (osd.4) 119 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:49 smithi155 bash[59495]: cluster 2023-10-09T06:32:48.274500+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e188 with expected crc 2023-10-09T06:32:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:49 smithi155 bash[59495]: audit 2023-10-09T06:32:48.587202+0000 mon.a (mon.0) 1041 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:49.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:49.452 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:32:49.453 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:32:49.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:49.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:49.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:49.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:49.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:49.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:50.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:50.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:50.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:50.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:50.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:50.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:50.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:50 smithi125 bash[77299]: cluster 2023-10-09T06:32:49.090944+0000 mgr.y (mgr.25723) 838 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 345 op/s 2023-10-09T06:32:50.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:50 smithi125 bash[80002]: cluster 2023-10-09T06:32:49.090944+0000 mgr.y (mgr.25723) 838 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 345 op/s 2023-10-09T06:32:50.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:50.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:50 smithi155 bash[59495]: cluster 2023-10-09T06:32:49.090944+0000 mgr.y (mgr.25723) 838 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 345 op/s 2023-10-09T06:32:50.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:50.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:50.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:50.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:50.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:50.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:50.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:50.872 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:32:50.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:32:50.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:50.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:50.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:50.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:51.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:51.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:51.232 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:32:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:32:51] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T06:32:51.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:51.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:51.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:51.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:51.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:51.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:51.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:51.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:51.883 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:32:51.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:32:51.929 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:32:51.950 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:32:51.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.972 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:32:51.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:51.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:51.995 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:32:52.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:32:52.052 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:32:52.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.053 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.054 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.054 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.054 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.054 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.054 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:32:52.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:52.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:52.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:52 smithi125 bash[80002]: cluster 2023-10-09T06:32:51.091870+0000 mgr.y (mgr.25723) 839 : cluster 0 pgmap v843: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 260 op/s 2023-10-09T06:32:52.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:52 smithi125 bash[80002]: audit 2023-10-09T06:32:51.854865+0000 mon.a (mon.0) 1042 : audit 0 from='client.? 172.21.15.125:0/3351414432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:52.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:52 smithi125 bash[77299]: cluster 2023-10-09T06:32:51.091870+0000 mgr.y (mgr.25723) 839 : cluster 0 pgmap v843: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 260 op/s 2023-10-09T06:32:52.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:52 smithi125 bash[77299]: audit 2023-10-09T06:32:51.854865+0000 mon.a (mon.0) 1042 : audit 0 from='client.? 172.21.15.125:0/3351414432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:52.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:52.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:52.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:52.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:52 smithi155 bash[59495]: cluster 2023-10-09T06:32:51.091870+0000 mgr.y (mgr.25723) 839 : cluster 0 pgmap v843: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 260 op/s 2023-10-09T06:32:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:52 smithi155 bash[59495]: audit 2023-10-09T06:32:51.854865+0000 mon.a (mon.0) 1042 : audit 0 from='client.? 172.21.15.125:0/3351414432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:52.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:32:52.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:52.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:52.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:52.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:52.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:52.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:52.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:52.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:52.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:52.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:53.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:53.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:53.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:53.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:53.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:53.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:53.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:53.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:53.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:53.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:53.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:53.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:53.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:53.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:53.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:53.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:53.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:53.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:54.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:54.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:54.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:54.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:54.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:54.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:54 smithi125 bash[77299]: cluster 2023-10-09T06:32:53.092791+0000 mgr.y (mgr.25723) 840 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 91 op/s 2023-10-09T06:32:54.376 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:54 smithi125 bash[80002]: cluster 2023-10-09T06:32:53.092791+0000 mgr.y (mgr.25723) 840 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 91 op/s 2023-10-09T06:32:54.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:54.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:54.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:54.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:54.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:54 smithi155 bash[59495]: cluster 2023-10-09T06:32:53.092791+0000 mgr.y (mgr.25723) 840 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 91 op/s 2023-10-09T06:32:54.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:54.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:54.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:54.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:55.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:55.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:55.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:55.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:55.203 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:32:55.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:55.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:55 smithi155 bash[59495]: audit 2023-10-09T06:32:54.906866+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.125:0/1330889607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.302 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:55 smithi155 bash[59495]: audit 2023-10-09T06:32:54.907366+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:55.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:55.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:55.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:55.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:55 smithi125 bash[77299]: audit 2023-10-09T06:32:54.906866+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.125:0/1330889607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:55 smithi125 bash[77299]: audit 2023-10-09T06:32:54.907366+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:55 smithi125 bash[80002]: audit 2023-10-09T06:32:54.906866+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.125:0/1330889607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:55 smithi125 bash[80002]: audit 2023-10-09T06:32:54.907366+0000 mon.a (mon.0) 1043 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:55.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:55.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:55.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:55.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:55.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:55.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:55.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:56.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:56.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:56.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:32:56.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: cluster 2023-10-09T06:32:55.093784+0000 mgr.y (mgr.25723) 841 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: cluster 2023-10-09T06:32:55.180861+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: audit 2023-10-09T06:32:55.192700+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: cluster 2023-10-09T06:32:55.192867+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: cluster 2023-10-09T06:32:55.195306+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.249 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:56 smithi155 bash[59495]: cluster 2023-10-09T06:32:55.195874+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:56.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:56.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:56.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: cluster 2023-10-09T06:32:55.093784+0000 mgr.y (mgr.25723) 841 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: cluster 2023-10-09T06:32:55.180861+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: audit 2023-10-09T06:32:55.192700+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: cluster 2023-10-09T06:32:55.192867+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: cluster 2023-10-09T06:32:55.195306+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.516 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[80002]: cluster 2023-10-09T06:32:55.195874+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: cluster 2023-10-09T06:32:55.093784+0000 mgr.y (mgr.25723) 841 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: cluster 2023-10-09T06:32:55.180861+0000 mon.a (mon.0) 1044 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: audit 2023-10-09T06:32:55.192700+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: cluster 2023-10-09T06:32:55.192867+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: cluster 2023-10-09T06:32:55.195306+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:56 smithi125 bash[77299]: cluster 2023-10-09T06:32:55.195874+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e189 with expected crc 2023-10-09T06:32:56.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:56.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:56.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:56.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:56.748 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:56.749 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:56.749 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:56.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:56.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:56.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:56.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:56.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:56.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:57.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:57.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:57.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:57.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:57.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:57.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:57.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:57.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:57.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:57.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:57.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:57.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:58.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:58.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:32:58.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.277 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:58 smithi125 bash[80002]: cluster 2023-10-09T06:32:57.094690+0000 mgr.y (mgr.25723) 842 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:32:58.277 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:58 smithi125 bash[77299]: cluster 2023-10-09T06:32:57.094690+0000 mgr.y (mgr.25723) 842 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:32:58.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:58.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:58 smithi155 bash[59495]: cluster 2023-10-09T06:32:57.094690+0000 mgr.y (mgr.25723) 842 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:32:58.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:58.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:58.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:58.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:58.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:58.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:58.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:58.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:59.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:59.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:59.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:59.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:59.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:59.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:59.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:59.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:59.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:59.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:59.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:59.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.977 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:32:59 smithi125 bash[77299]: cluster 2023-10-09T06:32:59.095385+0000 mgr.y (mgr.25723) 843 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:32:59.977 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:32:59 smithi125 bash[80002]: cluster 2023-10-09T06:32:59.095385+0000 mgr.y (mgr.25723) 843 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:32:59.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:32:59.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:00.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.105 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:32:59 smithi155 bash[59495]: cluster 2023-10-09T06:32:59.095385+0000 mgr.y (mgr.25723) 843 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:00.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:00.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:00.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:00.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:00.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:00.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:00.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:00.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:00.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:00.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:00.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:00.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:00.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:00.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:01.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:01.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:01.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:01.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:01.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:01] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-09T06:33:01.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:01.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:01.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:01.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:01.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:01.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:01.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:01.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:02.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:02.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:02.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:02.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:02.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:02.370 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:02 smithi125 bash[80002]: cluster 2023-10-09T06:33:01.096033+0000 mgr.y (mgr.25723) 844 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:02.370 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:02 smithi125 bash[77299]: cluster 2023-10-09T06:33:01.096033+0000 mgr.y (mgr.25723) 844 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:02.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:02 smithi155 bash[59495]: cluster 2023-10-09T06:33:01.096033+0000 mgr.y (mgr.25723) 844 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:02.395 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:02.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:02.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:02.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:02.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:02.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:02.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:02.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:02.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:02.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:03.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:03.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:03.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:03.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:03.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:03.443 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:03.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:03.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:03.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:03.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:03.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:03.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:03.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:03.939 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:03.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:03.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:04.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:04.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:04.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:04.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:04.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:04.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:04.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:04.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:04 smithi125 bash[77299]: cluster 2023-10-09T06:33:03.096747+0000 mgr.y (mgr.25723) 845 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:04.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:04 smithi125 bash[77299]: audit 2023-10-09T06:33:03.587346+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:04.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:04 smithi125 bash[80002]: cluster 2023-10-09T06:33:03.096747+0000 mgr.y (mgr.25723) 845 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:04.339 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:04 smithi125 bash[80002]: audit 2023-10-09T06:33:03.587346+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:04.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:04.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:04.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:04 smithi155 bash[59495]: cluster 2023-10-09T06:33:03.096747+0000 mgr.y (mgr.25723) 845 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:04.471 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:04 smithi155 bash[59495]: audit 2023-10-09T06:33:03.587346+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:04.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:04.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:04.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:04.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:04.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:04.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:04.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:04.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:04.954 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:05.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:05.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:05.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:05.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:05.173 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:33:05.174 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:33:05.174 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:33:05.174 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:33:05.174 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:33:05.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:05.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:05.378 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:05.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:05.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:05.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:05.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:05.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:05.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:05.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:05.787 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:05.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:05.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:05.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:06.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:06.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:06.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:06.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:06.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:06.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:06.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:06.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:06 smithi125 bash[77299]: cluster 2023-10-09T06:33:05.097737+0000 mgr.y (mgr.25723) 846 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:06.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:06 smithi125 bash[80002]: cluster 2023-10-09T06:33:05.097737+0000 mgr.y (mgr.25723) 846 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:06.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:06.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:06.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:06 smithi155 bash[59495]: cluster 2023-10-09T06:33:05.097737+0000 mgr.y (mgr.25723) 846 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:06.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:06.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:06.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:06.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:06.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:06.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:06.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:06.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:07.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:07.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:07.198 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_2' removed 2023-10-09T06:33:07.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:07.227 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:07 smithi155 bash[59495]: audit 2023-10-09T06:33:06.797113+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.125:0/3981534010' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.228 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:07 smithi155 bash[59495]: audit 2023-10-09T06:33:06.797757+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:07.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:07.361 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:07 smithi125 bash[77299]: audit 2023-10-09T06:33:06.797113+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.125:0/3981534010' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:07 smithi125 bash[77299]: audit 2023-10-09T06:33:06.797757+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:07 smithi125 bash[80002]: audit 2023-10-09T06:33:06.797113+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.125:0/3981534010' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:07 smithi125 bash[80002]: audit 2023-10-09T06:33:06.797757+0000 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:33:07.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:07.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:07.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:07.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:07.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:07.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:07.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:07.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:07.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:07.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:07.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:07.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:07.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:08.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:08.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:08.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:08.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:08.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.098325+0000 mgr.y (mgr.25723) 847 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:08.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: audit 2023-10-09T06:33:07.170239+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:33:08.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.170353+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:33:08.270 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.392739+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.392780+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.392780+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:08 smithi155 bash[59495]: cluster 2023-10-09T06:33:07.392783+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:08.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:08.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:33:08.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:08.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:08.431 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.098325+0000 mgr.y (mgr.25723) 847 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: audit 2023-10-09T06:33:07.170239+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.170353+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.392739+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.392780+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.392780+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[77299]: cluster 2023-10-09T06:33:07.392783+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.098325+0000 mgr.y (mgr.25723) 847 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:08.432 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: audit 2023-10-09T06:33:07.170239+0000 mon.a (mon.0) 1049 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:33:08.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.170353+0000 mon.a (mon.0) 1050 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:33:08.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.392739+0000 osd.4 (osd.4) 120 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.392780+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.392780+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:08 smithi125 bash[80002]: cluster 2023-10-09T06:33:07.392783+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e190 with expected crc 2023-10-09T06:33:08.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:08.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:08.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:08.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:08.767 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:33:08.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:08.769 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:33:08.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:08.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:08.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:08.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:08.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:09.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:09.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:09.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:09.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:09.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:09.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:09.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:09.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:09.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:09.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:09.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:09.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:09.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:09.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:10.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:10.104 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:09 smithi125 bash[80002]: audit 2023-10-09T06:33:08.749379+0000 mgr.y (mgr.25723) 848 : audit 0 from='client.45326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:10.105 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:09 smithi125 bash[80002]: cluster 2023-10-09T06:33:09.099085+0000 mgr.y (mgr.25723) 849 : cluster 0 pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:10.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:09 smithi125 bash[77299]: audit 2023-10-09T06:33:08.749379+0000 mgr.y (mgr.25723) 848 : audit 0 from='client.45326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:10.105 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:09 smithi125 bash[77299]: cluster 2023-10-09T06:33:09.099085+0000 mgr.y (mgr.25723) 849 : cluster 0 pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:10.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:10.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:10.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:09 smithi155 bash[59495]: audit 2023-10-09T06:33:08.749379+0000 mgr.y (mgr.25723) 848 : audit 0 from='client.45326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:10.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:09 smithi155 bash[59495]: cluster 2023-10-09T06:33:09.099085+0000 mgr.y (mgr.25723) 849 : cluster 0 pgmap v854: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:10.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:10.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:10.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:10.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:10.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:10.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:10.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:10.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:10.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:10.744 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:33:10.745 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:33:10.745 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T06:33:10.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T06:33:10.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:10.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:10.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:10.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:11.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:11.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:11.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:11.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:11.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:11.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:11.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:11.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:11.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:11.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:11.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:11.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:11] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T06:33:11.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:11.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:11.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:11.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:11.826 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:11.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:11.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:11.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:11.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:12.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:12.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:12.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:12.204 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:12 smithi125 bash[77299]: cluster 2023-10-09T06:33:11.099700+0000 mgr.y (mgr.25723) 850 : cluster 0 pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:12.204 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:12 smithi125 bash[80002]: cluster 2023-10-09T06:33:11.099700+0000 mgr.y (mgr.25723) 850 : cluster 0 pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:12.221 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:12.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:12.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:12.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:12 smithi155 bash[59495]: cluster 2023-10-09T06:33:11.099700+0000 mgr.y (mgr.25723) 850 : cluster 0 pgmap v855: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:12.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:12.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:12.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:12.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:12.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:12.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:12.802 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":853,"stamp":"2023-10-09T06:33:07.166259+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2760932,"kb_used_data":29028,"kb_used_omap":30,"kb_used_meta":2731809,"kb_avail":747200284,"statfs":{"total":767960285184,"available":765133090816,"internally_reserved":0,"allocated":29724672,"data_stored":24422526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":2797372527},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-209370,"num_write_kb":-6699777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.072131"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176369+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:32:48.176369+0000","last_peered":"2023-10-09T06:32:48.176369+0000","last_clean":"2023-10-09T06:32:48.176369+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:32:48.176369+0000","last_undegraded":"2023-10-09T06:32:48.176369+0000","last_fullsized":"2023-10-09T06:32:48.176369+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772651+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:32:48.772651+0000","last_peered":"2023-10-09T06:32:48.772651+0000","last_clean":"2023-10-09T06:32:48.772651+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:32:48.772651+0000","last_undegraded":"2023-10-09T06:32:48.772651+0000","last_fullsized":"2023-10-09T06:32:48.772651+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T06:32:55.664183+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:32:55.664183+0000","last_peered":"2023-10-09T06:32:55.664183+0000","last_clean":"2023-10-09T06:32:55.664183+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:32:55.664183+0000","last_undegraded":"2023-10-09T06:32:55.664183+0000","last_fullsized":"2023-10-09T06:32:55.664183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772743+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:32:48.772743+0000","last_peered":"2023-10-09T06:32:48.772743+0000","last_clean":"2023-10-09T06:32:48.772743+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:32:48.772743+0000","last_undegraded":"2023-10-09T06:32:48.772743+0000","last_fullsized":"2023-10-09T06:32:48.772743+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":180,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772182+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:32:48.772182+0000","last_peered":"2023-10-09T06:32:48.772182+0000","last_clean":"2023-10-09T06:32:48.772182+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:32:48.772182+0000","last_undegraded":"2023-10-09T06:32:48.772182+0000","last_fullsized":"2023-10-09T06:32:48.772182+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.279307+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:32:48.279307+0000","last_peered":"2023-10-09T06:32:48.279307+0000","last_clean":"2023-10-09T06:32:48.279307+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:32:48.279307+0000","last_undegraded":"2023-10-09T06:32:48.279307+0000","last_fullsized":"2023-10-09T06:32:48.279307+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":144,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772647+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:32:48.772647+0000","last_peered":"2023-10-09T06:32:48.772647+0000","last_clean":"2023-10-09T06:32:48.772647+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:32:48.772647+0000","last_undegraded":"2023-10-09T06:32:48.772647+0000","last_fullsized":"2023-10-09T06:32:48.772647+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772270+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:32:48.772270+0000","last_peered":"2023-10-09T06:32:48.772270+0000","last_clean":"2023-10-09T06:32:48.772270+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:32:48.772270+0000","last_undegraded":"2023-10-09T06:32:48.772270+0000","last_fullsized":"2023-10-09T06:32:48.772270+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779410+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:32:48.779410+0000","last_peered":"2023-10-09T06:32:48.779410+0000","last_clean":"2023-10-09T06:32:48.779410+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:32:48.779410+0000","last_undegraded":"2023-10-09T06:32:48.779410+0000","last_fullsized":"2023-10-09T06:32:48.779410+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":190,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773130+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:32:48.773130+0000","last_peered":"2023-10-09T06:32:48.773130+0000","last_clean":"2023-10-09T06:32:48.773130+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:32:48.773130+0000","last_undegraded":"2023-10-09T06:32:48.773130+0000","last_fullsized":"2023-10-09T06:32:48.773130+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":87,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778592+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:32:48.778592+0000","last_peered":"2023-10-09T06:32:48.778592+0000","last_clean":"2023-10-09T06:32:48.778592+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:32:48.778592+0000","last_undegraded":"2023-10-09T06:32:48.778592+0000","last_fullsized":"2023-10-09T06:32:48.778592+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772914+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:32:48.772914+0000","last_peered":"2023-10-09T06:32:48.772914+0000","last_clean":"2023-10-09T06:32:48.772914+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:32:48.772914+0000","last_undegraded":"2023-10-09T06:32:48.772914+0000","last_fullsized":"2023-10-09T06:32:48.772914+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":237,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176420+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:32:48.176420+0000","last_peered":"2023-10-09T06:32:48.176420+0000","last_clean":"2023-10-09T06:32:48.176420+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:32:48.176420+0000","last_undegraded":"2023-10-09T06:32:48.176420+0000","last_fullsized":"2023-10-09T06:32:48.176420+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.177172+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:32:48.177172+0000","last_peered":"2023-10-09T06:32:48.177172+0000","last_clean":"2023-10-09T06:32:48.177172+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:32:48.177172+0000","last_undegraded":"2023-10-09T06:32:48.177172+0000","last_fullsized":"2023-10-09T06:32:48.177172+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":87,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778339+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:32:48.778339+0000","last_peered":"2023-10-09T06:32:48.778339+0000","last_clean":"2023-10-09T06:32:48.778339+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:32:48.778339+0000","last_undegraded":"2023-10-09T06:32:48.778339+0000","last_fullsized":"2023-10-09T06:32:48.778339+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":182,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772780+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:32:48.772780+0000","last_peered":"2023-10-09T06:32:48.772780+0000","last_clean":"2023-10-09T06:32:48.772780+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:32:48.772780+0000","last_undegraded":"2023-10-09T06:32:48.772780+0000","last_fullsized":"2023-10-09T06:32:48.772780+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772200+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:32:48.772200+0000","last_peered":"2023-10-09T06:32:48.772200+0000","last_clean":"2023-10-09T06:32:48.772200+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:32:48.772200+0000","last_undegraded":"2023-10-09T06:32:48.772200+0000","last_fullsized":"2023-10-09T06:32:48.772200+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779971+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:32:48.779971+0000","last_peered":"2023-10-09T06:32:48.779971+0000","last_clean":"2023-10-09T06:32:48.779971+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:32:48.779971+0000","last_undegraded":"2023-10-09T06:32:48.779971+0000","last_fullsized":"2023-10-09T06:32:48.779971+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.777957+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:32:48.777957+0000","last_peered":"2023-10-09T06:32:48.777957+0000","last_clean":"2023-10-09T06:32:48.777957+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:32:48.777957+0000","last_undegraded":"2023-10-09T06:32:48.777957+0000","last_fullsized":"2023-10-09T06:32:48.777957+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":162,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772658+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:32:48.772658+0000","last_peered":"2023-10-09T06:32:48.772658+0000","last_clean":"2023-10-09T06:32:48.772658+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:32:48.772658+0000","last_undegraded":"2023-10-09T06:32:48.772658+0000","last_fullsized":"2023-10-09T06:32:48.772658+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":39,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.278801+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:32:48.278801+0000","last_peered":"2023-10-09T06:32:48.278801+0000","last_clean":"2023-10-09T06:32:48.278801+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:32:48.278801+0000","last_undegraded":"2023-10-09T06:32:48.278801+0000","last_fullsized":"2023-10-09T06:32:48.278801+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773047+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:32:48.773047+0000","last_peered":"2023-10-09T06:32:48.773047+0000","last_clean":"2023-10-09T06:32:48.773047+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:32:48.773047+0000","last_undegraded":"2023-10-09T06:32:48.773047+0000","last_fullsized":"2023-10-09T06:32:48.773047+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":85,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778728+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:32:48.778728+0000","last_peered":"2023-10-09T06:32:48.778728+0000","last_clean":"2023-10-09T06:32:48.778728+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:32:48.778728+0000","last_undegraded":"2023-10-09T06:32:48.778728+0000","last_fullsized":"2023-10-09T06:32:48.778728+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":174,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.777527+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:32:48.777527+0000","last_peered":"2023-10-09T06:32:48.777527+0000","last_clean":"2023-10-09T06:32:48.777527+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:32:48.777527+0000","last_undegraded":"2023-10-09T06:32:48.777527+0000","last_fullsized":"2023-10-09T06:32:48.777527+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":85,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778936+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:32:48.778936+0000","last_peered":"2023-10-09T06:32:48.778936+0000","last_clean":"2023-10-09T06:32:48.778936+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:32:48.778936+0000","last_undegraded":"2023-10-09T06:32:48.778936+0000","last_fullsized":"2023-10-09T06:32:48.778936+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176275+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:32:48.176275+0000","last_peered":"2023-10-09T06:32:48.176275+0000","last_clean":"2023-10-09T06:32:48.176275+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:32:48.176275+0000","last_undegraded":"2023-10-09T06:32:48.176275+0000","last_fullsized":"2023-10-09T06:32:48.176275+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":111,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778930+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:32:48.778930+0000","last_peered":"2023-10-09T06:32:48.778930+0000","last_clean":"2023-10-09T06:32:48.778930+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:32:48.778930+0000","last_undegraded":"2023-10-09T06:32:48.778930+0000","last_fullsized":"2023-10-09T06:32:48.778930+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.278786+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:32:48.278786+0000","last_peered":"2023-10-09T06:32:48.278786+0000","last_clean":"2023-10-09T06:32:48.278786+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:32:48.278786+0000","last_undegraded":"2023-10-09T06:32:48.278786+0000","last_fullsized":"2023-10-09T06:32:48.278786+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779286+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:32:48.779286+0000","last_peered":"2023-10-09T06:32:48.779286+0000","last_clean":"2023-10-09T06:32:48.779286+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:32:48.779286+0000","last_undegraded":"2023-10-09T06:32:48.779286+0000","last_fullsized":"2023-10-09T06:32:48.779286+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772276+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:32:48.772276+0000","last_peered":"2023-10-09T06:32:48.772276+0000","last_clean":"2023-10-09T06:32:48.772276+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:32:48.772276+0000","last_undegraded":"2023-10-09T06:32:48.772276+0000","last_fullsized":"2023-10-09T06:32:48.772276+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772498+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:32:48.772498+0000","last_peered":"2023-10-09T06:32:48.772498+0000","last_clean":"2023-10-09T06:32:48.772498+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:32:48.772498+0000","last_undegraded":"2023-10-09T06:32:48.772498+0000","last_fullsized":"2023-10-09T06:32:48.772498+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176285+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:32:48.176285+0000","last_peered":"2023-10-09T06:32:48.176285+0000","last_clean":"2023-10-09T06:32:48.176285+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:32:48.176285+0000","last_undegraded":"2023-10-09T06:32:48.176285+0000","last_fullsized":"2023-10-09T06:32:48.176285+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176945+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:32:48.176945+0000","last_peered":"2023-10-09T06:32:48.176945+0000","last_clean":"2023-10-09T06:32:48.176945+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:32:48.176945+0000","last_undegraded":"2023-10-09T06:32:48.176945+0000","last_fullsized":"2023-10-09T06:32:48.176945+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176666+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:32:48.176666+0000","last_peered":"2023-10-09T06:32:48.176666+0000","last_clean":"2023-10-09T06:32:48.176666+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:32:48.176666+0000","last_undegraded":"2023-10-09T06:32:48.176666+0000","last_fullsized":"2023-10-09T06:32:48.176666+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176127+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:32:48.176127+0000","last_peered":"2023-10-09T06:32:48.176127+0000","last_clean":"2023-10-09T06:32:48.176127+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:32:48.176127+0000","last_undegraded":"2023-10-09T06:32:48.176127+0000","last_fullsized":"2023-10-09T06:32:48.176127+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176729+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:32:48.176729+0000","last_peered":"2023-10-09T06:32:48.176729+0000","last_clean":"2023-10-09T06:32:48.176729+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:32:48.176729+0000","last_undegraded":"2023-10-09T06:32:48.176729+0000","last_fullsized":"2023-10-09T06:32:48.176729+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":184,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772372+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:32:48.772372+0000","last_peered":"2023-10-09T06:32:48.772372+0000","last_clean":"2023-10-09T06:32:48.772372+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:32:48.772372+0000","last_undegraded":"2023-10-09T06:32:48.772372+0000","last_fullsized":"2023-10-09T06:32:48.772372+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773417+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:32:48.773417+0000","last_peered":"2023-10-09T06:32:48.773417+0000","last_clean":"2023-10-09T06:32:48.773417+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:32:48.773417+0000","last_undegraded":"2023-10-09T06:32:48.773417+0000","last_fullsized":"2023-10-09T06:32:48.773417+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1042,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773391+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:32:48.773391+0000","last_peered":"2023-10-09T06:32:48.773391+0000","last_clean":"2023-10-09T06:32:48.773391+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:32:48.773391+0000","last_undegraded":"2023-10-09T06:32:48.773391+0000","last_fullsized":"2023-10-09T06:32:48.773391+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":169,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779442+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:32:48.779442+0000","last_peered":"2023-10-09T06:32:48.779442+0000","last_clean":"2023-10-09T06:32:48.779442+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:32:48.779442+0000","last_undegraded":"2023-10-09T06:32:48.779442+0000","last_fullsized":"2023-10-09T06:32:48.779442+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772515+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:32:48.772515+0000","last_peered":"2023-10-09T06:32:48.772515+0000","last_clean":"2023-10-09T06:32:48.772515+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:32:48.772515+0000","last_undegraded":"2023-10-09T06:32:48.772515+0000","last_fullsized":"2023-10-09T06:32:48.772515+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":827,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.777948+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:32:48.777948+0000","last_peered":"2023-10-09T06:32:48.777948+0000","last_clean":"2023-10-09T06:32:48.777948+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:32:48.777948+0000","last_undegraded":"2023-10-09T06:32:48.777948+0000","last_fullsized":"2023-10-09T06:32:48.777948+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":172,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.177123+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:32:48.177123+0000","last_peered":"2023-10-09T06:32:48.177123+0000","last_clean":"2023-10-09T06:32:48.177123+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:32:48.177123+0000","last_undegraded":"2023-10-09T06:32:48.177123+0000","last_fullsized":"2023-10-09T06:32:48.177123+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":157,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773267+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:32:48.773267+0000","last_peered":"2023-10-09T06:32:48.773267+0000","last_clean":"2023-10-09T06:32:48.773267+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:32:48.773267+0000","last_undegraded":"2023-10-09T06:32:48.773267+0000","last_fullsized":"2023-10-09T06:32:48.773267+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":553,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-10-09T06:32:55.552839+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:32:55.552839+0000","last_peered":"2023-10-09T06:32:55.552839+0000","last_clean":"2023-10-09T06:32:55.552839+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:32:55.552839+0000","last_undegraded":"2023-10-09T06:32:55.552839+0000","last_fullsized":"2023-10-09T06:32:55.552839+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.771772+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:32:48.771772+0000","last_peered":"2023-10-09T06:32:48.771772+0000","last_clean":"2023-10-09T06:32:48.771772+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:32:48.771772+0000","last_undegraded":"2023-10-09T06:32:48.771772+0000","last_fullsized":"2023-10-09T06:32:48.771772+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773418+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:32:48.773418+0000","last_peered":"2023-10-09T06:32:48.773418+0000","last_clean":"2023-10-09T06:32:48.773418+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:32:48.773418+0000","last_undegraded":"2023-10-09T06:32:48.773418+0000","last_fullsized":"2023-10-09T06:32:48.773418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":537,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779118+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:32:48.779118+0000","last_peered":"2023-10-09T06:32:48.779118+0000","last_clean":"2023-10-09T06:32:48.779118+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:32:48.779118+0000","last_undegraded":"2023-10-09T06:32:48.779118+0000","last_fullsized":"2023-10-09T06:32:48.779118+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":431,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778170+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:32:48.778170+0000","last_peered":"2023-10-09T06:32:48.778170+0000","last_clean":"2023-10-09T06:32:48.778170+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:32:48.778170+0000","last_undegraded":"2023-10-09T06:32:48.778170+0000","last_fullsized":"2023-10-09T06:32:48.778170+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":174,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773193+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:32:48.773193+0000","last_peered":"2023-10-09T06:32:48.773193+0000","last_clean":"2023-10-09T06:32:48.773193+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:32:48.773193+0000","last_undegraded":"2023-10-09T06:32:48.773193+0000","last_fullsized":"2023-10-09T06:32:48.773193+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":144,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176763+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:32:48.176763+0000","last_peered":"2023-10-09T06:32:48.176763+0000","last_clean":"2023-10-09T06:32:48.176763+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:32:48.176763+0000","last_undegraded":"2023-10-09T06:32:48.176763+0000","last_fullsized":"2023-10-09T06:32:48.176763+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":110,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.777586+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:32:48.777586+0000","last_peered":"2023-10-09T06:32:48.777586+0000","last_clean":"2023-10-09T06:32:48.777586+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:32:48.777586+0000","last_undegraded":"2023-10-09T06:32:48.777586+0000","last_fullsized":"2023-10-09T06:32:48.777586+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":145,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772049+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:32:48.772049+0000","last_peered":"2023-10-09T06:32:48.772049+0000","last_clean":"2023-10-09T06:32:48.772049+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:32:48.772049+0000","last_undegraded":"2023-10-09T06:32:48.772049+0000","last_fullsized":"2023-10-09T06:32:48.772049+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":160,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772992+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:32:48.772992+0000","last_peered":"2023-10-09T06:32:48.772992+0000","last_clean":"2023-10-09T06:32:48.772992+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:32:48.772992+0000","last_undegraded":"2023-10-09T06:32:48.772992+0000","last_fullsized":"2023-10-09T06:32:48.772992+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":99,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.778693+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:32:48.778693+0000","last_peered":"2023-10-09T06:32:48.778693+0000","last_clean":"2023-10-09T06:32:48.778693+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:32:48.778693+0000","last_undegraded":"2023-10-09T06:32:48.778693+0000","last_fullsized":"2023-10-09T06:32:48.778693+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773313+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:32:48.773313+0000","last_peered":"2023-10-09T06:32:48.773313+0000","last_clean":"2023-10-09T06:32:48.773313+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:32:48.773313+0000","last_undegraded":"2023-10-09T06:32:48.773313+0000","last_fullsized":"2023-10-09T06:32:48.773313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":157,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772521+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:32:48.772521+0000","last_peered":"2023-10-09T06:32:48.772521+0000","last_clean":"2023-10-09T06:32:48.772521+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:32:48.772521+0000","last_undegraded":"2023-10-09T06:32:48.772521+0000","last_fullsized":"2023-10-09T06:32:48.772521+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":190,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773096+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:32:48.773096+0000","last_peered":"2023-10-09T06:32:48.773096+0000","last_clean":"2023-10-09T06:32:48.773096+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:32:48.773096+0000","last_undegraded":"2023-10-09T06:32:48.773096+0000","last_fullsized":"2023-10-09T06:32:48.773096+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":571,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773527+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:32:48.773527+0000","last_peered":"2023-10-09T06:32:48.773527+0000","last_clean":"2023-10-09T06:32:48.773527+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:32:48.773527+0000","last_undegraded":"2023-10-09T06:32:48.773527+0000","last_fullsized":"2023-10-09T06:32:48.773527+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":189,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772080+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:32:48.772080+0000","last_peered":"2023-10-09T06:32:48.772080+0000","last_clean":"2023-10-09T06:32:48.772080+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:32:48.772080+0000","last_undegraded":"2023-10-09T06:32:48.772080+0000","last_fullsized":"2023-10-09T06:32:48.772080+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":709,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.279350+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:32:48.279350+0000","last_peered":"2023-10-09T06:32:48.279350+0000","last_clean":"2023-10-09T06:32:48.279350+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:32:48.279350+0000","last_undegraded":"2023-10-09T06:32:48.279350+0000","last_fullsized":"2023-10-09T06:32:48.279350+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":149,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772838+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:32:48.772838+0000","last_peered":"2023-10-09T06:32:48.772838+0000","last_clean":"2023-10-09T06:32:48.772838+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:32:48.772838+0000","last_undegraded":"2023-10-09T06:32:48.772838+0000","last_fullsized":"2023-10-09T06:32:48.772838+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176110+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:32:48.176110+0000","last_peered":"2023-10-09T06:32:48.176110+0000","last_clean":"2023-10-09T06:32:48.176110+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:32:48.176110+0000","last_undegraded":"2023-10-09T06:32:48.176110+0000","last_fullsized":"2023-10-09T06:32:48.176110+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":150,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772423+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:32:48.772423+0000","last_peered":"2023-10-09T06:32:48.772423+0000","last_clean":"2023-10-09T06:32:48.772423+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:32:48.772423+0000","last_undegraded":"2023-10-09T06:32:48.772423+0000","last_fullsized":"2023-10-09T06:32:48.772423+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.160784+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:32:48.160784+0000","last_peered":"2023-10-09T06:32:48.160784+0000","last_clean":"2023-10-09T06:32:48.160784+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:32:48.160784+0000","last_undegraded":"2023-10-09T06:32:48.160784+0000","last_fullsized":"2023-10-09T06:32:48.160784+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779529+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:32:48.779529+0000","last_peered":"2023-10-09T06:32:48.779529+0000","last_clean":"2023-10-09T06:32:48.779529+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:32:48.779529+0000","last_undegraded":"2023-10-09T06:32:48.779529+0000","last_fullsized":"2023-10-09T06:32:48.779529+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.177259+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:32:48.177259+0000","last_peered":"2023-10-09T06:32:48.177259+0000","last_clean":"2023-10-09T06:32:48.177259+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:32:48.177259+0000","last_undegraded":"2023-10-09T06:32:48.177259+0000","last_fullsized":"2023-10-09T06:32:48.177259+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":167,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.779613+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:32:48.779613+0000","last_peered":"2023-10-09T06:32:48.779613+0000","last_clean":"2023-10-09T06:32:48.779613+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:32:48.779613+0000","last_undegraded":"2023-10-09T06:32:48.779613+0000","last_fullsized":"2023-10-09T06:32:48.779613+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":168,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.176853+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:32:48.176853+0000","last_peered":"2023-10-09T06:32:48.176853+0000","last_clean":"2023-10-09T06:32:48.176853+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:32:48.176853+0000","last_undegraded":"2023-10-09T06:32:48.176853+0000","last_fullsized":"2023-10-09T06:32:48.176853+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":190,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772810+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:32:48.772810+0000","last_peered":"2023-10-09T06:32:48.772810+0000","last_clean":"2023-10-09T06:32:48.772810+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:32:48.772810+0000","last_undegraded":"2023-10-09T06:32:48.772810+0000","last_fullsized":"2023-10-09T06:32:48.772810+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":183,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.773214+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:32:48.773214+0000","last_peered":"2023-10-09T06:32:48.773214+0000","last_clean":"2023-10-09T06:32:48.773214+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:32:48.773214+0000","last_undegraded":"2023-10-09T06:32:48.773214+0000","last_fullsized":"2023-10-09T06:32:48.773214+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":149,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.772664+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:32:48.772664+0000","last_peered":"2023-10-09T06:32:48.772664+0000","last_clean":"2023-10-09T06:32:48.772664+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:32:48.772664+0000","last_undegraded":"2023-10-09T06:32:48.772664+0000","last_fullsized":"2023-10-09T06:32:48.772664+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-10-09T06:32:48.175772+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:32:48.175772+0000","last_peered":"2023-10-09T06:32:48.175772+0000","last_clean":"2023-10-09T06:32:48.175772+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:32:48.175772+0000","last_undegraded":"2023-10-09T06:32:48.175772+0000","last_fullsized":"2023-10-09T06:32:48.175772+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430621,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303648,"kb_used_data":2328,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441504,"statfs":{"total":95995035648,"available":95684100096,"internally_reserved":0,"allocated":2383872,"data_stored":1834522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":3.8969999999999998,"15min":2.9390000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":4.5510000000000002,"5min":178.738,"15min":178.738},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.2709999999999999,"5min":3.875,"15min":2.8759999999999999},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.252},"max":{"1min":4.2839999999999998,"5min":177.75,"15min":177.75},"last":0.746}]},{"osd":1,"last update":"Mon Oct 9 06:32:20 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.1499999999999999,"15min":2.0169999999999999},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":104.825},"last":1.073},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.343,"15min":1.927},"min":{"1min":0.32300000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6029999999999998,"5min":7.9119999999999999,"15min":104.637},"last":0.85299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.391,"15min":2.4289999999999998},"min":{"1min":0.36299999999999999,"5min":0.36199999999999999,"15min":0.29399999999999998},"max":{"1min":4.4489999999999998,"5min":15.585000000000001,"15min":104.992},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.542,"5min":1.375,"15min":2.3380000000000001},"min":{"1min":0.35399999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.0830000000000002,"5min":7.0629999999999997,"15min":104.691},"last":0.98799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:32:20 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":5.319,"15min":3.556},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":7.2279999999999998,"5min":183.971,"15min":183.971},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.593,"5min":5.1539999999999999,"15min":3.3570000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.9750000000000001,"5min":183.095,"15min":183.095},"last":1.21}]},{"osd":4,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.008,"15min":1.2270000000000001},"min":{"1min":0.372,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.3159999999999998,"5min":6.6239999999999997,"15min":11.029999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.296,"15min":1.5860000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.3050000000000002,"5min":15.504,"15min":15.504},"last":1.0169999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0489999999999999,"15min":1.4139999999999999},"min":{"1min":0.375,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.7130000000000001,"5min":7.1550000000000002,"15min":11.996},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":0.998,"15min":1.4059999999999999},"min":{"1min":0.42399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.6150000000000002,"5min":5.4909999999999997,"15min":13.634},"last":0.95699999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.4339999999999999,"15min":1.3240000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.2750000000000004,"5min":15.521000000000001,"15min":15.521000000000001},"last":1.052},{"interface":"front","average":{"1min":1.044,"5min":1.337,"15min":1.5049999999999999},"min":{"1min":0.42499999999999999,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":2.6389999999999998,"5min":15.449999999999999,"15min":15.449999999999999},"last":1.113}]}]},{"osd":6,"up_from":42,"seq":180388626853,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299536,"kb_used_data":2312,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93445616,"statfs":{"total":95995035648,"available":95688310784,"internally_reserved":0,"allocated":2367488,"data_stored":1833765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.2929999999999999,"15min":1.373},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":32.289999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.982,"5min":1.355,"15min":1.4830000000000001},"min":{"1min":0.42199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.9870000000000001,"5min":6.4240000000000004,"15min":31.454999999999998},"last":0.64900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:32:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.405,"15min":1.792},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":4.8390000000000004,"5min":6.9710000000000001,"15min":137.93899999999999},"last":0.876},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.4710000000000001,"15min":1.835},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":11.281000000000001,"5min":11.281000000000001,"15min":138.02500000000001},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:32:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4569999999999999,"5min":2.4780000000000002,"15min":2.387},"min":{"1min":0.37,"5min":0.35699999999999998,"15min":0.32000000000000001},"max":{"1min":26.477,"5min":53.616,"15min":95.540000000000006},"last":0.83099999999999996},{"interface":"front","average":{"1min":3.032,"5min":2.6139999999999999,"15min":2.395},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":25.745999999999999,"5min":53.665999999999997,"15min":96.352000000000004},"last":0.79400000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":3.423,"15min":2.3490000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":27.491,"5min":115.09399999999999,"15min":115.09399999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.0329999999999999,"15min":2.0099999999999998},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":26.815000000000001,"5min":112.886,"15min":112.886},"last":0.75800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:32:26 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93300000000000005,"15min":1.2749999999999999},"min":{"1min":0.376,"5min":0.314,"15min":0.26000000000000001},"max":{"1min":3.044,"5min":6.3799999999999999,"15min":7.3090000000000002},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.079,"15min":1.4239999999999999},"min":{"1min":0.35899999999999999,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":3.5529999999999999,"5min":7.0060000000000002,"15min":10.300000000000001},"last":1.0289999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:32:26 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":0.91600000000000004,"15min":1.4510000000000001},"min":{"1min":0.41899999999999998,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":3.145,"5min":6.4850000000000003,"15min":11.754},"last":1.052},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.119,"15min":1.5549999999999999},"min":{"1min":0.45900000000000002,"5min":0.38200000000000001,"15min":0.308},"max":{"1min":3.8029999999999999,"5min":12.215999999999999,"15min":12.295999999999999},"last":0.97099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.0920000000000001,"15min":1.492},"min":{"1min":0.47199999999999998,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":3.3069999999999999,"5min":6.5110000000000001,"15min":11.076000000000001},"last":1.149},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.0720000000000001,"15min":1.4199999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":2.8580000000000001,"5min":9.4179999999999993,"15min":11.102},"last":1.095}]}]},{"osd":1,"up_from":93,"seq":399431958762,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":332792,"kb_used_data":2032,"kb_used_omap":7,"kb_used_meta":330744,"kb_avail":93412360,"statfs":{"total":95995035648,"available":95654256640,"internally_reserved":0,"allocated":2080768,"data_stored":1278356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":338682256},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":6.173,"5min":3.4380000000000002,"15min":2.0489999999999999},"min":{"1min":0.216,"5min":0.187,"15min":0.187},"max":{"1min":89.555000000000007,"5min":89.555000000000007,"15min":89.555000000000007},"last":0.59099999999999997},{"interface":"front","average":{"1min":6.2539999999999996,"5min":3.3980000000000001,"15min":2.2400000000000002},"min":{"1min":0.27000000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":89.451999999999998,"5min":89.451999999999998,"15min":89.451999999999998},"last":0.63500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.637,"15min":1.9119999999999999},"min":{"1min":0.32900000000000001,"5min":0.214,"15min":0.19800000000000001},"max":{"1min":26.210000000000001,"5min":67.206000000000003,"15min":67.206000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":3.351,"5min":2.9209999999999998,"15min":1.978},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":26.026,"5min":67.132000000000005,"15min":67.132000000000005},"last":0.754}]},{"osd":3,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3380000000000001,"5min":4.2729999999999997,"15min":3.5790000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":26.184000000000001,"5min":73.337000000000003,"15min":127.44499999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.323,"5min":4.1669999999999998,"15min":3.5379999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":7.6699999999999999,"5min":73.388999999999996,"15min":123.959},"last":0.91400000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":1.635,"15min":1.508},"min":{"1min":0.372,"5min":0.372,"15min":0.30299999999999999},"max":{"1min":26.068999999999999,"5min":26.068999999999999,"15min":26.068999999999999},"last":0.86099999999999999},{"interface":"front","average":{"1min":1.242,"5min":1.4239999999999999,"15min":1.5389999999999999},"min":{"1min":0.36099999999999999,"5min":0.31,"15min":0.31},"max":{"1min":4.3979999999999997,"5min":5.6689999999999996,"15min":12.907999999999999},"last":0.98499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.5029999999999999,"15min":1.5900000000000001},"min":{"1min":0.35999999999999999,"5min":0.35899999999999999,"15min":0.32500000000000001},"max":{"1min":10.586,"5min":10.586,"15min":14.170999999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":2.5510000000000002,"5min":1.5489999999999999,"15min":1.643},"min":{"1min":0.39400000000000002,"5min":0.379,"15min":0.36599999999999999},"max":{"1min":26.157,"5min":26.157,"15min":26.157},"last":1.0529999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.5720000000000001,"15min":1.5149999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.36699999999999999},"max":{"1min":10.625,"5min":10.625,"15min":14.212999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.778,"5min":1.6759999999999999,"15min":1.645},"min":{"1min":0.35199999999999998,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":26.132000000000001,"5min":26.132000000000001,"15min":26.132000000000001},"last":1.0800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":1.9119999999999999,"15min":1.962},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.35399999999999998},"max":{"1min":26.109000000000002,"5min":26.109000000000002,"15min":30.774999999999999},"last":1.101},{"interface":"front","average":{"1min":1.54,"5min":1.472,"15min":1.6619999999999999},"min":{"1min":0.40400000000000003,"5min":0.40300000000000002,"15min":0.39700000000000002},"max":{"1min":5.806,"5min":5.8289999999999997,"15min":30.815000000000001},"last":1.1459999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122288,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389276,"kb_used_data":6676,"kb_used_omap":8,"kb_used_meta":382583,"kb_avail":93355876,"statfs":{"total":95995035648,"available":95596417024,"internally_reserved":0,"allocated":6836224,"data_stored":6058703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":391765248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:32:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8479999999999999,"5min":2.4910000000000001,"15min":2.5369999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.218},"max":{"1min":13.598000000000001,"5min":13.598000000000001,"15min":138.31700000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":3.2309999999999999,"5min":2.1469999999999998,"15min":2.3330000000000002},"min":{"1min":0.26700000000000002,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":12.101000000000001,"5min":12.101000000000001,"15min":144.37},"last":0.78800000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:32:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7890000000000001,"5min":2.5019999999999998,"15min":1.958},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":28.745000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":3.2599999999999998,"5min":2.0510000000000002,"15min":1.9199999999999999},"min":{"1min":0.22600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":16.129000000000001,"5min":16.129000000000001,"15min":28.146999999999998},"last":0.72199999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:32:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8809999999999998,"5min":3.2890000000000001,"15min":2.3250000000000002},"min":{"1min":0.34499999999999997,"5min":0.28699999999999998,"15min":0.20899999999999999},"max":{"1min":9.2720000000000002,"5min":56,"15min":59.643000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":3.419,"5min":3.0670000000000002,"15min":2.1840000000000002},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.791,"5min":54.524000000000001,"15min":59.762999999999998},"last":0.98099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:32:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.575,"15min":1.528},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":8.9920000000000009,"5min":8.9920000000000009,"15min":9.0890000000000004},"last":0.82999999999999996},{"interface":"front","average":{"1min":2.7010000000000001,"5min":1.7509999999999999,"15min":1.804},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.2319999999999993,"5min":9.2319999999999993,"15min":28.629000000000001},"last":0.69799999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:32:03 2023","interfaces":[{"interface":"back","average":{"1min":3.532,"5min":1.821,"15min":2.016},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.35399999999999998},"max":{"1min":12.356999999999999,"5min":12.356999999999999,"15min":28.574000000000002},"last":0.76200000000000001},{"interface":"front","average":{"1min":2.661,"5min":2.0299999999999998,"15min":1.8100000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.32600000000000001},"max":{"1min":7.6420000000000003,"5min":9.8209999999999997,"15min":28.061},"last":1.127}]},{"osd":6,"last update":"Mon Oct 9 06:32:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.9199999999999999,"15min":1.712},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.34599999999999997},"max":{"1min":16.181999999999999,"5min":16.181999999999999,"15min":27.966999999999999},"last":1.169},{"interface":"front","average":{"1min":2.0779999999999998,"5min":1.6439999999999999,"15min":1.4630000000000001},"min":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.33700000000000002},"max":{"1min":8.9600000000000009,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":0.89700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:32:03 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.9910000000000001,"15min":1.823},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":7.5759999999999996,"5min":9.1630000000000003,"15min":28.109999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":2.2839999999999998,"5min":1.909,"15min":1.849},"min":{"1min":0.45700000000000002,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":8.9190000000000005,"5min":8.9190000000000005,"15min":28.074999999999999},"last":0.99299999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795237,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408608,"kb_used_data":6104,"kb_used_omap":11,"kb_used_meta":402484,"kb_avail":93336544,"statfs":{"total":95995035648,"available":95576621056,"internally_reserved":0,"allocated":6250496,"data_stored":5466829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":412144574},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":3.0790000000000002,"15min":2.2869999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.17899999999999999},"max":{"1min":15.301,"5min":117.2,"15min":117.2},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.744,"5min":2.7679999999999998,"15min":2.3740000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":8.7449999999999992,"5min":121.521,"15min":121.521},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5880000000000001,"15min":1.6419999999999999},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":4.3799999999999999,"5min":12.737,"15min":12.737},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.4379999999999999,"15min":1.8420000000000001},"min":{"1min":0.26100000000000001,"5min":0.215,"15min":0.215},"max":{"1min":4.0439999999999996,"5min":10.997999999999999,"15min":11.943},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:32:18 2023","interfaces":[{"interface":"back","average":{"1min":3.96,"5min":6.5970000000000004,"15min":3.323},"min":{"1min":0.47399999999999998,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":11.493,"5min":201.33600000000001,"15min":201.33600000000001},"last":0.5},{"interface":"front","average":{"1min":2.4079999999999999,"5min":6.2140000000000004,"15min":3.2629999999999999},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":8.1880000000000006,"5min":199.95099999999999,"15min":199.95099999999999},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.3939999999999999,"15min":2.0059999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.339,"5min":7.5759999999999996,"15min":19.184000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.21,"15min":1.786},"min":{"1min":0.437,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":4.5819999999999999,"5min":6.3049999999999997,"15min":29.408000000000001},"last":0.78700000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.3570000000000002,"15min":2.3519999999999999},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.31},"max":{"1min":3.2759999999999998,"5min":86.423000000000002,"15min":86.423000000000002},"last":0.97499999999999998},{"interface":"front","average":{"1min":2.085,"5min":2.23,"15min":2.0129999999999999},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":12.532999999999999,"5min":86.382999999999996,"15min":86.382999999999996},"last":0.84799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.484,"15min":1.641},"min":{"1min":0.50800000000000001,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":8.2959999999999994,"5min":8.2959999999999994,"15min":29.504000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.879,"5min":1.4470000000000001,"15min":1.673},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":6.343,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.94599999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.591,"15min":1.7969999999999999},"min":{"1min":0.42599999999999999,"5min":0.41599999999999998,"15min":0.308},"max":{"1min":9.2050000000000001,"5min":9.2050000000000001,"15min":9.2050000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.579,"5min":1.5860000000000001,"15min":2.0129999999999999},"min":{"1min":0.39300000000000002,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":4.2380000000000004,"5min":7.6920000000000002,"15min":12.051},"last":1.107}]}]},{"osd":3,"up_from":103,"seq":442381631711,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408676,"kb_used_data":6108,"kb_used_omap":3,"kb_used_meta":402556,"kb_avail":93336476,"statfs":{"total":95995035648,"available":95576551424,"internally_reserved":0,"allocated":6254592,"data_stored":5467050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":412217889},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3460000000000001,"5min":2.2829999999999999,"15min":2.4790000000000001},"min":{"1min":0.32400000000000001,"5min":0.184,"15min":0.16900000000000001},"max":{"1min":6.2610000000000001,"5min":15.285,"15min":147.988},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.2719999999999998,"5min":1.9019999999999999,"15min":2.302},"min":{"1min":0.19,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.3380000000000001,"5min":9.625,"15min":147.94200000000001},"last":0.64100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.6660000000000004,"5min":2.5089999999999999,"15min":1.661},"min":{"1min":0.31,"5min":0.19,"15min":0.19},"max":{"1min":47.057000000000002,"5min":47.057000000000002,"15min":47.057000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":4.7839999999999998,"5min":3.0070000000000001,"15min":1.966},"min":{"1min":0.29299999999999998,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":54.752000000000002,"5min":54.752000000000002,"15min":54.752000000000002},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5190000000000001,"5min":3.012,"15min":2.3940000000000001},"min":{"1min":0.36699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.3659999999999997,"5min":47.097999999999999,"15min":70.594999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.4990000000000001,"15min":2},"min":{"1min":0.32400000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.683999999999999,"5min":46.997,"15min":62.991999999999997},"last":0.77300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":2.444,"15min":1.891},"min":{"1min":0.315,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.2889999999999997,"5min":72.653000000000006,"15min":72.653000000000006},"last":0.88400000000000001},{"interface":"front","average":{"1min":2.661,"5min":2.5630000000000002,"15min":2.0289999999999999},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":11.756,"5min":72.706999999999994,"15min":72.706999999999994},"last":1.022}]},{"osd":5,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.982,"15min":1.73},"min":{"1min":0.41199999999999998,"5min":0.35999999999999999,"15min":0.35499999999999998},"max":{"1min":11.781000000000001,"5min":11.781000000000001,"15min":11.781000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.77,"5min":1.792,"15min":1.7450000000000001},"min":{"1min":0.34899999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.8789999999999996,"5min":9.4619999999999997,"15min":11.754},"last":0.91800000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":2.0049999999999999,"15min":1.649},"min":{"1min":0.49299999999999999,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":5.577,"5min":15.122999999999999,"15min":15.122999999999999},"last":1.373},{"interface":"front","average":{"1min":2.8039999999999998,"5min":2.0529999999999999,"15min":1.645},"min":{"1min":0.39900000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":11.728999999999999,"5min":11.728999999999999,"15min":11.728999999999999},"last":1.3140000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4660000000000002,"5min":1.9450000000000001,"15min":1.746},"min":{"1min":0.45200000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.3090000000000002,"5min":6.2149999999999999,"15min":18.695},"last":1.109},{"interface":"front","average":{"1min":1.8859999999999999,"5min":2.1480000000000001,"15min":1.8819999999999999},"min":{"1min":0.50800000000000001,"5min":0.43099999999999999,"15min":0.39900000000000002},"max":{"1min":5.5359999999999996,"5min":13.211,"15min":13.211},"last":1.2430000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019319,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307160,"kb_used_data":1744,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437992,"statfs":{"total":95995035648,"available":95680503808,"internally_reserved":0,"allocated":1785856,"data_stored":1244084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":2.488,"15min":2.452},"min":{"1min":0.28999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":16.218,"5min":115.59699999999999,"15min":115.59699999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.6659999999999999,"5min":2.6499999999999999,"15min":2.778},"min":{"1min":0.33500000000000002,"5min":0.28000000000000003,"15min":0.27700000000000002},"max":{"1min":6.4880000000000004,"5min":115.65000000000001,"15min":115.65000000000001},"last":0.91700000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":1.228,"15min":2.1819999999999999},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.23999999999999999},"max":{"1min":7.0499999999999998,"5min":7.0499999999999998,"15min":179.47300000000001},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.702,"5min":1.496,"15min":2.5979999999999999},"min":{"1min":0.38300000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.9199999999999999,"5min":9.9909999999999997,"15min":179.447},"last":1.1819999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.6319999999999999,"15min":2.153},"min":{"1min":0.38300000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.705,"5min":14.705,"15min":61.783999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.6020000000000001,"15min":2.29},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.7850000000000001,"5min":9.7759999999999998,"15min":61.551000000000002},"last":1.0049999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:32:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":2.161,"15min":2.3799999999999999},"min":{"1min":0.34899999999999998,"5min":0.34300000000000003,"15min":0.33000000000000002},"max":{"1min":8.1449999999999996,"5min":12.081,"15min":91.361999999999995},"last":1.089},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.794,"15min":1.9690000000000001},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":8.1859999999999999,"5min":12.064,"15min":89.676000000000002},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:32:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":0.83999999999999997,"15min":1.298},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.737,"5min":11.737,"15min":12.242000000000001},"last":0.876},{"interface":"front","average":{"1min":1.24,"5min":1.0680000000000001,"15min":1.6899999999999999},"min":{"1min":0.47799999999999998,"5min":0.36199999999999999,"15min":0.26500000000000001},"max":{"1min":4.5060000000000002,"5min":4.5060000000000002,"15min":20.113},"last":0.82499999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.337,"15min":1.5920000000000001},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.29499999999999998},"max":{"1min":6.5990000000000002,"5min":8.1110000000000007,"15min":18.010000000000002},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.268,"15min":1.4350000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.782,"5min":11.782,"15min":13.913},"last":1.1220000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.464,"15min":1.7749999999999999},"min":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.371},"max":{"1min":6.8310000000000004,"5min":12.779,"15min":15.294},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.734,"5min":1.2250000000000001,"15min":1.7569999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.1370000000000005,"5min":9.1370000000000005,"15min":14.896000000000001},"last":1.234}]}]},{"osd":5,"up_from":36,"seq":154618823086,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311236,"kb_used_data":1724,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433916,"statfs":{"total":95995035648,"available":95676329984,"internally_reserved":0,"allocated":1765376,"data_stored":1239217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:50 2023","interfaces":[{"interface":"back","average":{"1min":5.0469999999999997,"5min":2.9220000000000002,"15min":2.7589999999999999},"min":{"1min":0.26700000000000002,"5min":0.254,"15min":0.247},"max":{"1min":83.197000000000003,"5min":83.197000000000003,"15min":83.197000000000003},"last":1.038},{"interface":"front","average":{"1min":5.4960000000000004,"5min":2.7639999999999998,"15min":2.4710000000000001},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":83.563999999999993,"5min":83.563999999999993,"15min":83.563999999999993},"last":0.56899999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":1.3300000000000001,"15min":2.3940000000000001},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.4779999999999998,"5min":5.9299999999999997,"15min":161.71199999999999},"last":0.747},{"interface":"front","average":{"1min":1.397,"5min":1.516,"15min":2.6480000000000001},"min":{"1min":0.33600000000000002,"5min":0.31,"15min":0.29399999999999998},"max":{"1min":4.806,"5min":9.0589999999999993,"15min":162.50200000000001},"last":0.71899999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":2.1160000000000001,"15min":2.2839999999999998},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":5.1050000000000004,"5min":54.981999999999999,"15min":54.981999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.9159999999999999,"5min":2.1749999999999998,"15min":2.2570000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":9.8889999999999993,"5min":55.008000000000003,"15min":59.082999999999998},"last":1.2150000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:32:15 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":3.2690000000000001,"15min":2.5870000000000002},"min":{"1min":0.44700000000000001,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":6.3570000000000002,"5min":70.186000000000007,"15min":70.186000000000007},"last":0.89700000000000002},{"interface":"front","average":{"1min":2.2629999999999999,"5min":3.117,"15min":2.3849999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1029999999999998,"5min":63.691000000000003,"15min":65.792000000000002},"last":1.071}]},{"osd":4,"last update":"Mon Oct 9 06:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.208,"15min":1.879},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1980000000000004,"5min":8.8879999999999999,"15min":17.667999999999999},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.117,"15min":1.599},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3040000000000003,"5min":5.3040000000000003,"15min":16.181999999999999},"last":1.1220000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:32:15 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.2290000000000001,"15min":1.6890000000000001},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.35699999999999998},"max":{"1min":4.1550000000000002,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.506,"5min":1.234,"15min":1.629},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":6.4219999999999997,"5min":9.3450000000000006,"15min":12.365},"last":1.296}]},{"osd":7,"last update":"Mon Oct 9 06:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.0840000000000001,"15min":1.702},"min":{"1min":0.47599999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.7039999999999997,"5min":5.7039999999999997,"15min":13.047000000000001},"last":1.252},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.214,"15min":1.5349999999999999},"min":{"1min":0.42499999999999999,"5min":0.41599999999999998,"15min":0.36799999999999999},"max":{"1min":5.5919999999999996,"5min":5.5919999999999996,"15min":11.981},"last":1.3109999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:33:12.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:33:12.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:12.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:12.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:12.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.012 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:13.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:13.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:13.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:13.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:13.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:13.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:13.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:13.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:13.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:13.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:13.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:13.796 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:13.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:13.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:13.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:14.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:14.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:14.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:14.197 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:14 smithi125 bash[77299]: cluster 2023-10-09T06:33:13.100508+0000 mgr.y (mgr.25723) 851 : cluster 0 pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:14.197 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:14 smithi125 bash[80002]: cluster 2023-10-09T06:33:13.100508+0000 mgr.y (mgr.25723) 851 : cluster 0 pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:14.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:14.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:14.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:14.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.452 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:14 smithi155 bash[59495]: cluster 2023-10-09T06:33:13.100508+0000 mgr.y (mgr.25723) 851 : cluster 0 pgmap v856: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:14.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:14.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:14.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.748 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:14.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:14.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:14.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:14.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:15.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:15.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:15.201 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:33:15.227 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:33:15.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:15.251 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:33:15.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:15.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:15.274 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:33:15.303 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:33:15.330 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:33:15.360 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:33:15.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:15.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:15 smithi155 bash[59495]: audit 2023-10-09T06:33:15.152160+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.125:0/1358057871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:15.386 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:33:15.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.387 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.388 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:33:15.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:15.439 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:15 smithi125 bash[77299]: audit 2023-10-09T06:33:15.152160+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.125:0/1358057871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:15.441 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:15 smithi125 bash[80002]: audit 2023-10-09T06:33:15.152160+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.125:0/1358057871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:15.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:15.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:15.676 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:15.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:15.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:15.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:15.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:15.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:16.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:16.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:16.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:16.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[80002]: cluster 2023-10-09T06:33:15.101194+0000 mgr.y (mgr.25723) 852 : cluster 0 pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:16.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[80002]: audit 2023-10-09T06:33:15.428745+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:16.187 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[80002]: audit 2023-10-09T06:33:15.522056+0000 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:16.187 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[80002]: audit 2023-10-09T06:33:15.805154+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.187 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[80002]: audit 2023-10-09T06:33:15.813841+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.189 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:33:16.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:16.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:16.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:16 smithi155 bash[59495]: cluster 2023-10-09T06:33:15.101194+0000 mgr.y (mgr.25723) 852 : cluster 0 pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:16 smithi155 bash[59495]: audit 2023-10-09T06:33:15.428745+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:16 smithi155 bash[59495]: audit 2023-10-09T06:33:15.522056+0000 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:16 smithi155 bash[59495]: audit 2023-10-09T06:33:15.805154+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:16 smithi155 bash[59495]: audit 2023-10-09T06:33:15.813841+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:16.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:16.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[77299]: cluster 2023-10-09T06:33:15.101194+0000 mgr.y (mgr.25723) 852 : cluster 0 pgmap v857: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:16.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[77299]: audit 2023-10-09T06:33:15.428745+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:16.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[77299]: audit 2023-10-09T06:33:15.522056+0000 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:16.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[77299]: audit 2023-10-09T06:33:15.805154+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:16 smithi125 bash[77299]: audit 2023-10-09T06:33:15.813841+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:16.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:16.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:16.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:16.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:16.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:16.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:16.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:16.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:16.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:16.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:17.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:17.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:17.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:17.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:17.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:17.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:17.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[77299]: cluster 2023-10-09T06:33:16.169779+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:17.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[77299]: cluster 2023-10-09T06:33:16.169812+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T06:33:17.294 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[77299]: audit 2023-10-09T06:33:16.176984+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[77299]: cluster 2023-10-09T06:33:16.177059+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[77299]: cluster 2023-10-09T06:33:16.178730+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[80002]: cluster 2023-10-09T06:33:16.169779+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[80002]: cluster 2023-10-09T06:33:16.169812+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[80002]: audit 2023-10-09T06:33:16.176984+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:17.295 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[80002]: cluster 2023-10-09T06:33:16.177059+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:33:17.296 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:17 smithi125 bash[80002]: cluster 2023-10-09T06:33:16.178730+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:17.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:17.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:17 smithi155 bash[59495]: cluster 2023-10-09T06:33:16.169779+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:17.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:17 smithi155 bash[59495]: cluster 2023-10-09T06:33:16.169812+0000 mon.a (mon.0) 1056 : cluster 1 Cluster is now healthy 2023-10-09T06:33:17.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:17 smithi155 bash[59495]: audit 2023-10-09T06:33:16.176984+0000 mon.a (mon.0) 1057 : audit 1 from='client.? 172.21.15.125:0/3352576140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:17.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:17 smithi155 bash[59495]: cluster 2023-10-09T06:33:16.177059+0000 mon.a (mon.0) 1058 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:33:17.381 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:17 smithi155 bash[59495]: cluster 2023-10-09T06:33:16.178730+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:17.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:17.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:17.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:17.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:17.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:17.727 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:17.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:17.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:17.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:17.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:17.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:17.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:18.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:18.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:18.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.333 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[77299]: cluster 2023-10-09T06:33:17.102082+0000 mgr.y (mgr.25723) 853 : cluster 0 pgmap v859: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[77299]: cluster 2023-10-09T06:33:17.183652+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[77299]: cluster 2023-10-09T06:33:17.186320+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[77299]: cluster 2023-10-09T06:33:17.485027+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[77299]: cluster 2023-10-09T06:33:17.660574+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[80002]: cluster 2023-10-09T06:33:17.102082+0000 mgr.y (mgr.25723) 853 : cluster 0 pgmap v859: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[80002]: cluster 2023-10-09T06:33:17.183652+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.334 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[80002]: cluster 2023-10-09T06:33:17.186320+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[80002]: cluster 2023-10-09T06:33:17.485027+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:18 smithi125 bash[80002]: cluster 2023-10-09T06:33:17.660574+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:18.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:18.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:18 smithi155 bash[59495]: cluster 2023-10-09T06:33:17.102082+0000 mgr.y (mgr.25723) 853 : cluster 0 pgmap v859: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:18.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:18 smithi155 bash[59495]: cluster 2023-10-09T06:33:17.183652+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:18 smithi155 bash[59495]: cluster 2023-10-09T06:33:17.186320+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:18 smithi155 bash[59495]: cluster 2023-10-09T06:33:17.485027+0000 osd.4 (osd.4) 121 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:18 smithi155 bash[59495]: cluster 2023-10-09T06:33:17.660574+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e191 with expected crc 2023-10-09T06:33:18.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:33:18.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:18.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:18.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:18.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:18.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:18.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:18.786 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:18.787 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:18.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:18.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:18.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:18.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:19.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:19.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:19.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:19.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:33:19.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:19.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.438 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:19.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:19.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:19.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:19 smithi155 bash[59495]: audit 2023-10-09T06:33:18.591610+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:19.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:19 smithi155 bash[59495]: audit 2023-10-09T06:33:18.592806+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:19.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:19 smithi155 bash[59495]: cluster 2023-10-09T06:33:19.102687+0000 mgr.y (mgr.25723) 854 : cluster 0 pgmap v860: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:19.710 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:19.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:19.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:19.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:19.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:19.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:20.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:20.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:20.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[77299]: audit 2023-10-09T06:33:18.591610+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:20.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[77299]: audit 2023-10-09T06:33:18.592806+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:20.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[77299]: cluster 2023-10-09T06:33:19.102687+0000 mgr.y (mgr.25723) 854 : cluster 0 pgmap v860: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:20.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[80002]: audit 2023-10-09T06:33:18.591610+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:20.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[80002]: audit 2023-10-09T06:33:18.592806+0000 mon.a (mon.0) 1060 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:20.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:19 smithi125 bash[80002]: cluster 2023-10-09T06:33:19.102687+0000 mgr.y (mgr.25723) 854 : cluster 0 pgmap v860: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:20.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:20.250 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:20.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:20.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:20.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:20.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:20.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:20.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:20.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:20.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:20.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:20.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:20.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:20.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:21.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:21.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:21.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:21.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:21] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:33:21.441 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:21.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:21.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:21.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:21.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:21.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:21.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:21.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:21.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:22.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:22.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:22.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:22.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:22.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:22.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:22.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[77299]: cluster 2023-10-09T06:33:21.103393+0000 mgr.y (mgr.25723) 855 : cluster 0 pgmap v861: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[77299]: audit 2023-10-09T06:33:21.673044+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[77299]: audit 2023-10-09T06:33:21.684167+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[80002]: cluster 2023-10-09T06:33:21.103393+0000 mgr.y (mgr.25723) 855 : cluster 0 pgmap v861: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[80002]: audit 2023-10-09T06:33:21.673044+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:22 smithi125 bash[80002]: audit 2023-10-09T06:33:21.684167+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:22 smithi155 bash[59495]: cluster 2023-10-09T06:33:21.103393+0000 mgr.y (mgr.25723) 855 : cluster 0 pgmap v861: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:22.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:22 smithi155 bash[59495]: audit 2023-10-09T06:33:21.673044+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:22 smithi155 bash[59495]: audit 2023-10-09T06:33:21.684167+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:22.411 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:22.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:22.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:22.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:22.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:22.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:22.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:22.910 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:22.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:22.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.060 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:23.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.189 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:23.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:23.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:23.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.714 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:23.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:23.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:23.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:24.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:24.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:24.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:24.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:24.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:24.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:24.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:24 smithi125 bash[77299]: cluster 2023-10-09T06:33:23.104086+0000 mgr.y (mgr.25723) 856 : cluster 0 pgmap v862: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:24.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:24 smithi125 bash[80002]: cluster 2023-10-09T06:33:23.104086+0000 mgr.y (mgr.25723) 856 : cluster 0 pgmap v862: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:24.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:24.502 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:24 smithi155 bash[59495]: cluster 2023-10-09T06:33:23.104086+0000 mgr.y (mgr.25723) 856 : cluster 0 pgmap v862: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:24.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:24.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:24.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:24.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:24.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:24.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:24.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:24.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:24.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:25.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:25.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:25.139 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:25.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:25.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:25.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:25.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:25.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:25.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:25.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:25.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:25.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:25.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:26.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:26.112 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:26.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:26.229 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:26 smithi125 bash[77299]: cluster 2023-10-09T06:33:25.105053+0000 mgr.y (mgr.25723) 857 : cluster 0 pgmap v863: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:26.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:26 smithi125 bash[80002]: cluster 2023-10-09T06:33:25.105053+0000 mgr.y (mgr.25723) 857 : cluster 0 pgmap v863: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:26.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:26.338 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:26.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:26.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:26.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:26.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:26.603 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:26.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:26 smithi155 bash[59495]: cluster 2023-10-09T06:33:25.105053+0000 mgr.y (mgr.25723) 857 : cluster 0 pgmap v863: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:26.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:26.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:26.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:26.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:26.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:27.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:27.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:27.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:27.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:27.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:27.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:27.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:27.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:27.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:27.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:27.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:27.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:27.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:27.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:27.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:28.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:28.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:28.156 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:33:28.156 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:33:28.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:28.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:28.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:28.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: cluster 2023-10-09T06:33:27.105491+0000 mgr.y (mgr.25723) 858 : cluster 0 pgmap v864: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: audit 2023-10-09T06:33:27.366016+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: audit 2023-10-09T06:33:27.371888+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: audit 2023-10-09T06:33:27.793159+0000 mon.a (mon.0) 1065 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: audit 2023-10-09T06:33:27.794170+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:28.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:28 smithi155 bash[59495]: audit 2023-10-09T06:33:27.800134+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: cluster 2023-10-09T06:33:27.105491+0000 mgr.y (mgr.25723) 858 : cluster 0 pgmap v864: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:28.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: audit 2023-10-09T06:33:27.366016+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.432 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: audit 2023-10-09T06:33:27.371888+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: audit 2023-10-09T06:33:27.793159+0000 mon.a (mon.0) 1065 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: audit 2023-10-09T06:33:27.794170+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[77299]: audit 2023-10-09T06:33:27.800134+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: cluster 2023-10-09T06:33:27.105491+0000 mgr.y (mgr.25723) 858 : cluster 0 pgmap v864: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: audit 2023-10-09T06:33:27.366016+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: audit 2023-10-09T06:33:27.371888+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: audit 2023-10-09T06:33:27.793159+0000 mon.a (mon.0) 1065 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:28.433 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: audit 2023-10-09T06:33:27.794170+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:28.434 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:28 smithi125 bash[80002]: audit 2023-10-09T06:33:27.800134+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:28.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:28.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:28.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:28.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:28.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:28.901 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:28.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:28.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:28.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:29.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:29.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:29.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:29.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:29.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:29.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:29.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:29.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:29.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:29.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:29.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:29.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:29.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:29.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:29.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:29.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:29.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:29.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:29.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:29.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:30.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:30.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:30.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:29 smithi125 bash[77299]: cluster 2023-10-09T06:33:29.106185+0000 mgr.y (mgr.25723) 859 : cluster 0 pgmap v865: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:30.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:29 smithi125 bash[80002]: cluster 2023-10-09T06:33:29.106185+0000 mgr.y (mgr.25723) 859 : cluster 0 pgmap v865: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:30.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:30.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:30.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:30.370 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:29 smithi155 bash[59495]: cluster 2023-10-09T06:33:29.106185+0000 mgr.y (mgr.25723) 859 : cluster 0 pgmap v865: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:30.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:30.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:30.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:30.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:30.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:30.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:30.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:30.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:30.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:30.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:31.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:31.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:31.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:31.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:31.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:31.413 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:31.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:31.456 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:31] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:33:31.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.568 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:31.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:31.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:31.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:31.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:31.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:31.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:32.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:32.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:32.185 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_3' created 2023-10-09T06:33:32.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:32 smithi125 bash[77299]: cluster 2023-10-09T06:33:31.106686+0000 mgr.y (mgr.25723) 860 : cluster 0 pgmap v866: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:32.219 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:32 smithi125 bash[77299]: audit 2023-10-09T06:33:31.203995+0000 mon.a (mon.0) 1068 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:33:32.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:32 smithi125 bash[80002]: cluster 2023-10-09T06:33:31.106686+0000 mgr.y (mgr.25723) 860 : cluster 0 pgmap v866: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:32.219 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:32 smithi125 bash[80002]: audit 2023-10-09T06:33:31.203995+0000 mon.a (mon.0) 1068 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:33:32.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:32.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:32 smithi155 bash[59495]: cluster 2023-10-09T06:33:31.106686+0000 mgr.y (mgr.25723) 860 : cluster 0 pgmap v866: 73 pgs: 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:32 smithi155 bash[59495]: audit 2023-10-09T06:33:31.203995+0000 mon.a (mon.0) 1068 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:33:32.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:33:32.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:32.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:32.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:32.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:32.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:32.825 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:33:32.827 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:33:32.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:32.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:32.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:32.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:33.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:33.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:33.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:33.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:33.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: audit 2023-10-09T06:33:32.171065+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: cluster 2023-10-09T06:33:32.171158+0000 mon.a (mon.0) 1070 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: cluster 2023-10-09T06:33:32.173564+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: cluster 2023-10-09T06:33:32.177906+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: cluster 2023-10-09T06:33:32.178806+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:33 smithi155 bash[59495]: cluster 2023-10-09T06:33:32.181480+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:33.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: audit 2023-10-09T06:33:32.171065+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: cluster 2023-10-09T06:33:32.171158+0000 mon.a (mon.0) 1070 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: cluster 2023-10-09T06:33:32.173564+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: cluster 2023-10-09T06:33:32.177906+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: cluster 2023-10-09T06:33:32.178806+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[77299]: cluster 2023-10-09T06:33:32.181480+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: audit 2023-10-09T06:33:32.171065+0000 mon.a (mon.0) 1069 : audit 1 from='client.? 172.21.15.125:0/4177431464' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: cluster 2023-10-09T06:33:32.171158+0000 mon.a (mon.0) 1070 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:33:33.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: cluster 2023-10-09T06:33:32.173564+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: cluster 2023-10-09T06:33:32.177906+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: cluster 2023-10-09T06:33:32.178806+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:33 smithi125 bash[80002]: cluster 2023-10-09T06:33:32.181480+0000 osd.4 (osd.4) 122 : cluster 3 failed to encode map e192 with expected crc 2023-10-09T06:33:33.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:33.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:33.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:33.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:33.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:33.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:33.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:34.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:34.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:34.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:34.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:34.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:34.323 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: audit 2023-10-09T06:33:32.805177+0000 mgr.y (mgr.25723) 861 : audit 0 from='client.45350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:34.323 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.107180+0000 mgr.y (mgr.25723) 862 : cluster 0 pgmap v868: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.180666+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.181734+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.185967+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.187075+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: cluster 2023-10-09T06:33:33.187193+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.324 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:34 smithi155 bash[59495]: audit 2023-10-09T06:33:33.588651+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:34.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:34.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: audit 2023-10-09T06:33:32.805177+0000 mgr.y (mgr.25723) 861 : audit 0 from='client.45350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:34.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.107180+0000 mgr.y (mgr.25723) 862 : cluster 0 pgmap v868: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:34.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.180666+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.181734+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.185967+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.187075+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: cluster 2023-10-09T06:33:33.187193+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[77299]: audit 2023-10-09T06:33:33.588651+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: audit 2023-10-09T06:33:32.805177+0000 mgr.y (mgr.25723) 861 : audit 0 from='client.45350 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.107180+0000 mgr.y (mgr.25723) 862 : cluster 0 pgmap v868: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.180666+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:33:34.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.181734+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.185967+0000 osd.6 (osd.6) 130 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.187075+0000 osd.4 (osd.4) 123 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: cluster 2023-10-09T06:33:33.187193+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-09T06:33:34.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:34 smithi125 bash[80002]: audit 2023-10-09T06:33:33.588651+0000 mon.a (mon.0) 1072 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:34.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:34.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:34.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:34.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:34.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:34.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:34.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:34.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:34.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:34.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:34.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:35.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:35.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T06:33:35.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:35.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:35.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:35.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:35.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:35.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:35.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:35 smithi125 bash[80002]: cluster 2023-10-09T06:33:34.188714+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:33:35.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:35 smithi125 bash[80002]: cluster 2023-10-09T06:33:34.191329+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T06:33:35.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:35 smithi125 bash[77299]: cluster 2023-10-09T06:33:34.188714+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:33:35.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:35 smithi125 bash[77299]: cluster 2023-10-09T06:33:34.191329+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T06:33:35.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:35.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:35 smithi155 bash[59495]: cluster 2023-10-09T06:33:34.188714+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:33:35.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:35 smithi155 bash[59495]: cluster 2023-10-09T06:33:34.191329+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e194 with expected crc 2023-10-09T06:33:35.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:35.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:35.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:35.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:35.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:35.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:35.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:35.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:35.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:36.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:36.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:36.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:36.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:36.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:36.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:36.304 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:36 smithi155 bash[59495]: cluster 2023-10-09T06:33:35.108027+0000 mgr.y (mgr.25723) 863 : cluster 0 pgmap v871: 89 pgs: 4 creating+peering, 3 unknown, 82 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:36.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:36.426 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:36.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:36.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:36.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:36.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:36 smithi125 bash[80002]: cluster 2023-10-09T06:33:35.108027+0000 mgr.y (mgr.25723) 863 : cluster 0 pgmap v871: 89 pgs: 4 creating+peering, 3 unknown, 82 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:36.578 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:36 smithi125 bash[77299]: cluster 2023-10-09T06:33:35.108027+0000 mgr.y (mgr.25723) 863 : cluster 0 pgmap v871: 89 pgs: 4 creating+peering, 3 unknown, 82 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:36.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:36.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:36.705 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":867,"stamp":"2023-10-09T06:33:32.168198+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":1106,"num_write_kb":15159,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2783268,"kb_used_data":29220,"kb_used_omap":30,"kb_used_meta":2753953,"kb_avail":747177948,"statfs":{"total":767960285184,"available":765110218752,"internally_reserved":0,"allocated":29921280,"data_stored":24546798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":2820047983},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.065293"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":172,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.398412+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:33:07.398412+0000","last_peered":"2023-10-09T06:33:07.398412+0000","last_clean":"2023-10-09T06:33:07.398412+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:33:07.398412+0000","last_undegraded":"2023-10-09T06:33:07.398412+0000","last_fullsized":"2023-10-09T06:33:07.398412+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.183321+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:33:07.183321+0000","last_peered":"2023-10-09T06:33:07.183321+0000","last_clean":"2023-10-09T06:33:07.183321+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:33:07.183321+0000","last_undegraded":"2023-10-09T06:33:07.183321+0000","last_fullsized":"2023-10-09T06:33:07.183321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":142,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.288619+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:33:07.288619+0000","last_peered":"2023-10-09T06:33:07.288619+0000","last_clean":"2023-10-09T06:33:07.288619+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:33:07.288619+0000","last_undegraded":"2023-10-09T06:33:07.288619+0000","last_fullsized":"2023-10-09T06:33:07.288619+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:17.261754+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:33:17.261754+0000","last_peered":"2023-10-09T06:33:17.261754+0000","last_clean":"2023-10-09T06:33:17.261754+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:33:17.261754+0000","last_undegraded":"2023-10-09T06:33:17.261754+0000","last_fullsized":"2023-10-09T06:33:17.261754+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":182,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.168800+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:33:07.168800+0000","last_peered":"2023-10-09T06:33:07.168800+0000","last_clean":"2023-10-09T06:33:07.168800+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:33:07.168800+0000","last_undegraded":"2023-10-09T06:33:07.168800+0000","last_fullsized":"2023-10-09T06:33:07.168800+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":31,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.413846+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:33:07.413846+0000","last_peered":"2023-10-09T06:33:07.413846+0000","last_clean":"2023-10-09T06:33:07.413846+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:33:07.413846+0000","last_undegraded":"2023-10-09T06:33:07.413846+0000","last_fullsized":"2023-10-09T06:33:07.413846+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.363925+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:33:07.363925+0000","last_peered":"2023-10-09T06:33:07.363925+0000","last_clean":"2023-10-09T06:33:07.363925+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:33:07.363925+0000","last_undegraded":"2023-10-09T06:33:07.363925+0000","last_fullsized":"2023-10-09T06:33:07.363925+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:16.972808+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:33:16.972808+0000","last_peered":"2023-10-09T06:33:16.972808+0000","last_clean":"2023-10-09T06:33:16.972808+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:33:16.972808+0000","last_undegraded":"2023-10-09T06:33:16.972808+0000","last_fullsized":"2023-10-09T06:33:16.972808+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":174,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406966+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:33:07.406966+0000","last_peered":"2023-10-09T06:33:07.406966+0000","last_clean":"2023-10-09T06:33:07.406966+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:33:07.406966+0000","last_undegraded":"2023-10-09T06:33:07.406966+0000","last_fullsized":"2023-10-09T06:33:07.406966+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":193,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:16.972896+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:33:16.972896+0000","last_peered":"2023-10-09T06:33:16.972896+0000","last_clean":"2023-10-09T06:33:16.972896+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:33:16.972896+0000","last_undegraded":"2023-10-09T06:33:16.972896+0000","last_fullsized":"2023-10-09T06:33:16.972896+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":89,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.402442+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:33:07.402442+0000","last_peered":"2023-10-09T06:33:07.402442+0000","last_clean":"2023-10-09T06:33:07.402442+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:33:07.402442+0000","last_undegraded":"2023-10-09T06:33:07.402442+0000","last_fullsized":"2023-10-09T06:33:07.402442+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":161,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.290069+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:33:07.290069+0000","last_peered":"2023-10-09T06:33:07.290069+0000","last_clean":"2023-10-09T06:33:07.290069+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:33:07.290069+0000","last_undegraded":"2023-10-09T06:33:07.290069+0000","last_fullsized":"2023-10-09T06:33:07.290069+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":239,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.401826+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:33:07.401826+0000","last_peered":"2023-10-09T06:33:07.401826+0000","last_clean":"2023-10-09T06:33:07.401826+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:33:07.401826+0000","last_undegraded":"2023-10-09T06:33:07.401826+0000","last_fullsized":"2023-10-09T06:33:07.401826+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400308+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:33:07.400308+0000","last_peered":"2023-10-09T06:33:07.400308+0000","last_clean":"2023-10-09T06:33:07.400308+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:33:07.400308+0000","last_undegraded":"2023-10-09T06:33:07.400308+0000","last_fullsized":"2023-10-09T06:33:07.400308+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":89,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.398751+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:33:07.398751+0000","last_peered":"2023-10-09T06:33:07.398751+0000","last_clean":"2023-10-09T06:33:07.398751+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:33:07.398751+0000","last_undegraded":"2023-10-09T06:33:07.398751+0000","last_fullsized":"2023-10-09T06:33:07.398751+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":185,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:16.973336+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:33:16.973336+0000","last_peered":"2023-10-09T06:33:16.973336+0000","last_clean":"2023-10-09T06:33:16.973336+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:33:16.973336+0000","last_undegraded":"2023-10-09T06:33:16.973336+0000","last_fullsized":"2023-10-09T06:33:16.973336+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.358802+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:33:07.358802+0000","last_peered":"2023-10-09T06:33:07.358802+0000","last_clean":"2023-10-09T06:33:07.358802+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:33:07.358802+0000","last_undegraded":"2023-10-09T06:33:07.358802+0000","last_fullsized":"2023-10-09T06:33:07.358802+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406134+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:33:07.406134+0000","last_peered":"2023-10-09T06:33:07.406134+0000","last_clean":"2023-10-09T06:33:07.406134+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:33:07.406134+0000","last_undegraded":"2023-10-09T06:33:07.406134+0000","last_fullsized":"2023-10-09T06:33:07.406134+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":101,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.399026+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:33:07.399026+0000","last_peered":"2023-10-09T06:33:07.399026+0000","last_clean":"2023-10-09T06:33:07.399026+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:33:07.399026+0000","last_undegraded":"2023-10-09T06:33:07.399026+0000","last_fullsized":"2023-10-09T06:33:07.399026+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.364845+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:33:07.364845+0000","last_peered":"2023-10-09T06:33:07.364845+0000","last_clean":"2023-10-09T06:33:07.364845+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:33:07.364845+0000","last_undegraded":"2023-10-09T06:33:07.364845+0000","last_fullsized":"2023-10-09T06:33:07.364845+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":41,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.402553+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:33:07.402553+0000","last_peered":"2023-10-09T06:33:07.402553+0000","last_clean":"2023-10-09T06:33:07.402553+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:33:07.402553+0000","last_undegraded":"2023-10-09T06:33:07.402553+0000","last_fullsized":"2023-10-09T06:33:07.402553+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":197,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.184934+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:33:07.184934+0000","last_peered":"2023-10-09T06:33:07.184934+0000","last_clean":"2023-10-09T06:33:07.184934+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:33:07.184934+0000","last_undegraded":"2023-10-09T06:33:07.184934+0000","last_fullsized":"2023-10-09T06:33:07.184934+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":87,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.401545+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:33:07.401545+0000","last_peered":"2023-10-09T06:33:07.401545+0000","last_clean":"2023-10-09T06:33:07.401545+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:33:07.401545+0000","last_undegraded":"2023-10-09T06:33:07.401545+0000","last_fullsized":"2023-10-09T06:33:07.401545+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":176,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.408456+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:33:07.408456+0000","last_peered":"2023-10-09T06:33:07.408456+0000","last_clean":"2023-10-09T06:33:07.408456+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:33:07.408456+0000","last_undegraded":"2023-10-09T06:33:07.408456+0000","last_fullsized":"2023-10-09T06:33:07.408456+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":87,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400697+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:33:07.400697+0000","last_peered":"2023-10-09T06:33:07.400697+0000","last_clean":"2023-10-09T06:33:07.400697+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:33:07.400697+0000","last_undegraded":"2023-10-09T06:33:07.400697+0000","last_fullsized":"2023-10-09T06:33:07.400697+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.401877+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:33:07.401877+0000","last_peered":"2023-10-09T06:33:07.401877+0000","last_clean":"2023-10-09T06:33:07.401877+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:33:07.401877+0000","last_undegraded":"2023-10-09T06:33:07.401877+0000","last_fullsized":"2023-10-09T06:33:07.401877+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":113,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.402618+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:33:07.402618+0000","last_peered":"2023-10-09T06:33:07.402618+0000","last_clean":"2023-10-09T06:33:07.402618+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:33:07.402618+0000","last_undegraded":"2023-10-09T06:33:07.402618+0000","last_fullsized":"2023-10-09T06:33:07.402618+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":31,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.402603+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:33:07.402603+0000","last_peered":"2023-10-09T06:33:07.402603+0000","last_clean":"2023-10-09T06:33:07.402603+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:33:07.402603+0000","last_undegraded":"2023-10-09T06:33:07.402603+0000","last_fullsized":"2023-10-09T06:33:07.402603+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406151+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:33:07.406151+0000","last_peered":"2023-10-09T06:33:07.406151+0000","last_clean":"2023-10-09T06:33:07.406151+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:33:07.406151+0000","last_undegraded":"2023-10-09T06:33:07.406151+0000","last_fullsized":"2023-10-09T06:33:07.406151+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:17.261339+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:33:17.261339+0000","last_peered":"2023-10-09T06:33:17.261339+0000","last_clean":"2023-10-09T06:33:17.261339+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:33:17.261339+0000","last_undegraded":"2023-10-09T06:33:17.261339+0000","last_fullsized":"2023-10-09T06:33:17.261339+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":166,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.178465+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:33:07.178465+0000","last_peered":"2023-10-09T06:33:07.178465+0000","last_clean":"2023-10-09T06:33:07.178465+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:33:07.178465+0000","last_undegraded":"2023-10-09T06:33:07.178465+0000","last_fullsized":"2023-10-09T06:33:07.178465+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.398230+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:33:07.398230+0000","last_peered":"2023-10-09T06:33:07.398230+0000","last_clean":"2023-10-09T06:33:07.398230+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:33:07.398230+0000","last_undegraded":"2023-10-09T06:33:07.398230+0000","last_fullsized":"2023-10-09T06:33:07.398230+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":121,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.398632+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:33:07.398632+0000","last_peered":"2023-10-09T06:33:07.398632+0000","last_clean":"2023-10-09T06:33:07.398632+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:33:07.398632+0000","last_undegraded":"2023-10-09T06:33:07.398632+0000","last_fullsized":"2023-10-09T06:33:07.398632+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400305+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:33:07.400305+0000","last_peered":"2023-10-09T06:33:07.400305+0000","last_clean":"2023-10-09T06:33:07.400305+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:33:07.400305+0000","last_undegraded":"2023-10-09T06:33:07.400305+0000","last_fullsized":"2023-10-09T06:33:07.400305+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.401589+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:33:07.401589+0000","last_peered":"2023-10-09T06:33:07.401589+0000","last_clean":"2023-10-09T06:33:07.401589+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:33:07.401589+0000","last_undegraded":"2023-10-09T06:33:07.401589+0000","last_fullsized":"2023-10-09T06:33:07.401589+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400043+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:33:07.400043+0000","last_peered":"2023-10-09T06:33:07.400043+0000","last_clean":"2023-10-09T06:33:07.400043+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:33:07.400043+0000","last_undegraded":"2023-10-09T06:33:07.400043+0000","last_fullsized":"2023-10-09T06:33:07.400043+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.168979+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:33:07.168979+0000","last_peered":"2023-10-09T06:33:07.168979+0000","last_clean":"2023-10-09T06:33:07.168979+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:33:07.168979+0000","last_undegraded":"2023-10-09T06:33:07.168979+0000","last_fullsized":"2023-10-09T06:33:07.168979+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.288645+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:33:07.288645+0000","last_peered":"2023-10-09T06:33:07.288645+0000","last_clean":"2023-10-09T06:33:07.288645+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:33:07.288645+0000","last_undegraded":"2023-10-09T06:33:07.288645+0000","last_fullsized":"2023-10-09T06:33:07.288645+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1044,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.290585+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:33:07.290585+0000","last_peered":"2023-10-09T06:33:07.290585+0000","last_clean":"2023-10-09T06:33:07.290585+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:33:07.290585+0000","last_undegraded":"2023-10-09T06:33:07.290585+0000","last_fullsized":"2023-10-09T06:33:07.290585+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":171,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406243+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:33:07.406243+0000","last_peered":"2023-10-09T06:33:07.406243+0000","last_clean":"2023-10-09T06:33:07.406243+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:33:07.406243+0000","last_undegraded":"2023-10-09T06:33:07.406243+0000","last_fullsized":"2023-10-09T06:33:07.406243+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.289511+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:33:07.289511+0000","last_peered":"2023-10-09T06:33:07.289511+0000","last_clean":"2023-10-09T06:33:07.289511+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:33:07.289511+0000","last_undegraded":"2023-10-09T06:33:07.289511+0000","last_fullsized":"2023-10-09T06:33:07.289511+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":829,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.409213+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:33:07.409213+0000","last_peered":"2023-10-09T06:33:07.409213+0000","last_clean":"2023-10-09T06:33:07.409213+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:33:07.409213+0000","last_undegraded":"2023-10-09T06:33:07.409213+0000","last_fullsized":"2023-10-09T06:33:07.409213+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":174,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400061+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:33:07.400061+0000","last_peered":"2023-10-09T06:33:07.400061+0000","last_clean":"2023-10-09T06:33:07.400061+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:33:07.400061+0000","last_undegraded":"2023-10-09T06:33:07.400061+0000","last_fullsized":"2023-10-09T06:33:07.400061+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":159,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.289356+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:33:07.289356+0000","last_peered":"2023-10-09T06:33:07.289356+0000","last_clean":"2023-10-09T06:33:07.289356+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:33:07.289356+0000","last_undegraded":"2023-10-09T06:33:07.289356+0000","last_fullsized":"2023-10-09T06:33:07.289356+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":555,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.167929+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:33:07.167929+0000","last_peered":"2023-10-09T06:33:07.167929+0000","last_clean":"2023-10-09T06:33:07.167929+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:33:07.167929+0000","last_undegraded":"2023-10-09T06:33:07.167929+0000","last_fullsized":"2023-10-09T06:33:07.167929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":161,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.288341+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:33:07.288341+0000","last_peered":"2023-10-09T06:33:07.288341+0000","last_clean":"2023-10-09T06:33:07.288341+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:33:07.288341+0000","last_undegraded":"2023-10-09T06:33:07.288341+0000","last_fullsized":"2023-10-09T06:33:07.288341+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:12.397959+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:33:12.397959+0000","last_peered":"2023-10-09T06:33:12.397959+0000","last_clean":"2023-10-09T06:33:12.397959+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:33:12.397959+0000","last_undegraded":"2023-10-09T06:33:12.397959+0000","last_fullsized":"2023-10-09T06:33:12.397959+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":539,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406616+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:33:07.406616+0000","last_peered":"2023-10-09T06:33:07.406616+0000","last_clean":"2023-10-09T06:33:07.406616+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:33:07.406616+0000","last_undegraded":"2023-10-09T06:33:07.406616+0000","last_fullsized":"2023-10-09T06:33:07.406616+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":433,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.399836+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:33:07.399836+0000","last_peered":"2023-10-09T06:33:07.399836+0000","last_clean":"2023-10-09T06:33:07.399836+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:33:07.399836+0000","last_undegraded":"2023-10-09T06:33:07.399836+0000","last_fullsized":"2023-10-09T06:33:07.399836+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":177,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-10-09T06:33:16.972964+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:33:16.972964+0000","last_peered":"2023-10-09T06:33:16.972964+0000","last_clean":"2023-10-09T06:33:16.972964+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:33:16.972964+0000","last_undegraded":"2023-10-09T06:33:16.972964+0000","last_fullsized":"2023-10-09T06:33:16.972964+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400513+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:33:07.400513+0000","last_peered":"2023-10-09T06:33:07.400513+0000","last_clean":"2023-10-09T06:33:07.400513+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:33:07.400513+0000","last_undegraded":"2023-10-09T06:33:07.400513+0000","last_fullsized":"2023-10-09T06:33:07.400513+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":112,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.397373+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:33:07.397373+0000","last_peered":"2023-10-09T06:33:07.397373+0000","last_clean":"2023-10-09T06:33:07.397373+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:33:07.397373+0000","last_undegraded":"2023-10-09T06:33:07.397373+0000","last_fullsized":"2023-10-09T06:33:07.397373+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.358583+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:33:07.358583+0000","last_peered":"2023-10-09T06:33:07.358583+0000","last_clean":"2023-10-09T06:33:07.358583+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:33:07.358583+0000","last_undegraded":"2023-10-09T06:33:07.358583+0000","last_fullsized":"2023-10-09T06:33:07.358583+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":162,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.365309+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:33:07.365309+0000","last_peered":"2023-10-09T06:33:07.365309+0000","last_clean":"2023-10-09T06:33:07.365309+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:33:07.365309+0000","last_undegraded":"2023-10-09T06:33:07.365309+0000","last_fullsized":"2023-10-09T06:33:07.365309+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":101,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.402531+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:33:07.402531+0000","last_peered":"2023-10-09T06:33:07.402531+0000","last_clean":"2023-10-09T06:33:07.402531+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:33:07.402531+0000","last_undegraded":"2023-10-09T06:33:07.402531+0000","last_fullsized":"2023-10-09T06:33:07.402531+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.289275+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:33:07.289275+0000","last_peered":"2023-10-09T06:33:07.289275+0000","last_clean":"2023-10-09T06:33:07.289275+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:33:07.289275+0000","last_undegraded":"2023-10-09T06:33:07.289275+0000","last_fullsized":"2023-10-09T06:33:07.289275+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.359014+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:33:07.359014+0000","last_peered":"2023-10-09T06:33:07.359014+0000","last_clean":"2023-10-09T06:33:07.359014+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:33:07.359014+0000","last_undegraded":"2023-10-09T06:33:07.359014+0000","last_fullsized":"2023-10-09T06:33:07.359014+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":192,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.184079+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:33:07.184079+0000","last_peered":"2023-10-09T06:33:07.184079+0000","last_clean":"2023-10-09T06:33:07.184079+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:33:07.184079+0000","last_undegraded":"2023-10-09T06:33:07.184079+0000","last_fullsized":"2023-10-09T06:33:07.184079+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":574,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.397421+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:33:07.397421+0000","last_peered":"2023-10-09T06:33:07.397421+0000","last_clean":"2023-10-09T06:33:07.397421+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:33:07.397421+0000","last_undegraded":"2023-10-09T06:33:07.397421+0000","last_fullsized":"2023-10-09T06:33:07.397421+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.168692+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:33:07.168692+0000","last_peered":"2023-10-09T06:33:07.168692+0000","last_clean":"2023-10-09T06:33:07.168692+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:33:07.168692+0000","last_undegraded":"2023-10-09T06:33:07.168692+0000","last_fullsized":"2023-10-09T06:33:07.168692+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:33:32.159382+0000","last_change":"2023-10-09T06:33:32.159382+0000","last_active":"2023-10-09T06:33:32.159382+0000","last_peered":"2023-10-09T06:33:32.159382+0000","last_clean":"2023-10-09T06:33:32.159382+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:33:32.159382+0000","last_undegraded":"2023-10-09T06:33:32.159382+0000","last_fullsized":"2023-10-09T06:33:32.159382+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":711,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.419279+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:33:07.419279+0000","last_peered":"2023-10-09T06:33:07.419279+0000","last_clean":"2023-10-09T06:33:07.419279+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:33:07.419279+0000","last_undegraded":"2023-10-09T06:33:07.419279+0000","last_fullsized":"2023-10-09T06:33:07.419279+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":151,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.364973+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:33:07.364973+0000","last_peered":"2023-10-09T06:33:07.364973+0000","last_clean":"2023-10-09T06:33:07.364973+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:33:07.364973+0000","last_undegraded":"2023-10-09T06:33:07.364973+0000","last_fullsized":"2023-10-09T06:33:07.364973+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.401711+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:33:07.401711+0000","last_peered":"2023-10-09T06:33:07.401711+0000","last_clean":"2023-10-09T06:33:07.401711+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:33:07.401711+0000","last_undegraded":"2023-10-09T06:33:07.401711+0000","last_fullsized":"2023-10-09T06:33:07.401711+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":153,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:13.704120+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:33:13.704120+0000","last_peered":"2023-10-09T06:33:13.704120+0000","last_clean":"2023-10-09T06:33:13.704120+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:33:13.704120+0000","last_undegraded":"2023-10-09T06:33:13.704120+0000","last_fullsized":"2023-10-09T06:33:13.704120+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.168202+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:33:07.168202+0000","last_peered":"2023-10-09T06:33:07.168202+0000","last_clean":"2023-10-09T06:33:07.168202+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:33:07.168202+0000","last_undegraded":"2023-10-09T06:33:07.168202+0000","last_fullsized":"2023-10-09T06:33:07.168202+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":158,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406320+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:33:07.406320+0000","last_peered":"2023-10-09T06:33:07.406320+0000","last_clean":"2023-10-09T06:33:07.406320+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:33:07.406320+0000","last_undegraded":"2023-10-09T06:33:07.406320+0000","last_fullsized":"2023-10-09T06:33:07.406320+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.400818+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:33:07.400818+0000","last_peered":"2023-10-09T06:33:07.400818+0000","last_clean":"2023-10-09T06:33:07.400818+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:33:07.400818+0000","last_undegraded":"2023-10-09T06:33:07.400818+0000","last_fullsized":"2023-10-09T06:33:07.400818+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":169,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.406352+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:33:07.406352+0000","last_peered":"2023-10-09T06:33:07.406352+0000","last_clean":"2023-10-09T06:33:07.406352+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:33:07.406352+0000","last_undegraded":"2023-10-09T06:33:07.406352+0000","last_fullsized":"2023-10-09T06:33:07.406352+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.398839+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:33:07.398839+0000","last_peered":"2023-10-09T06:33:07.398839+0000","last_clean":"2023-10-09T06:33:07.398839+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:33:07.398839+0000","last_undegraded":"2023-10-09T06:33:07.398839+0000","last_fullsized":"2023-10-09T06:33:07.398839+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":192,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.183808+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:33:07.183808+0000","last_peered":"2023-10-09T06:33:07.183808+0000","last_clean":"2023-10-09T06:33:07.183808+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:33:07.183808+0000","last_undegraded":"2023-10-09T06:33:07.183808+0000","last_fullsized":"2023-10-09T06:33:07.183808+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.168880+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:33:07.168880+0000","last_peered":"2023-10-09T06:33:07.168880+0000","last_clean":"2023-10-09T06:33:07.168880+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:33:07.168880+0000","last_undegraded":"2023-10-09T06:33:07.168880+0000","last_fullsized":"2023-10-09T06:33:07.168880+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":151,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.364003+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:33:07.364003+0000","last_peered":"2023-10-09T06:33:07.364003+0000","last_clean":"2023-10-09T06:33:07.364003+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:33:07.364003+0000","last_undegraded":"2023-10-09T06:33:07.364003+0000","last_fullsized":"2023-10-09T06:33:07.364003+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-10-09T06:33:07.397658+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:33:07.397658+0000","last_peered":"2023-10-09T06:33:07.397658+0000","last_clean":"2023-10-09T06:33:07.397658+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:33:07.397658+0000","last_undegraded":"2023-10-09T06:33:07.397658+0000","last_fullsized":"2023-10-09T06:33:07.397658+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430626,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303668,"kb_used_data":2348,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441484,"statfs":{"total":95995035648,"available":95684079616,"internally_reserved":0,"allocated":2404352,"data_stored":1849721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":3.8969999999999998,"15min":2.9390000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":4.5510000000000002,"5min":178.738,"15min":178.738},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.2709999999999999,"5min":3.875,"15min":2.8759999999999999},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.252},"max":{"1min":4.2839999999999998,"5min":177.75,"15min":177.75},"last":0.72399999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.2070000000000001,"15min":2.0329999999999999},"min":{"1min":0.35699999999999998,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":4.2190000000000003,"5min":7.4740000000000002,"15min":104.825},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.101,"5min":1.3819999999999999,"15min":1.9350000000000001},"min":{"1min":0.67900000000000005,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5030000000000001,"5min":7.9119999999999999,"15min":104.637},"last":0.77300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.391,"15min":2.4289999999999998},"min":{"1min":0.36299999999999999,"5min":0.36199999999999999,"15min":0.29399999999999998},"max":{"1min":4.4489999999999998,"5min":15.585000000000001,"15min":104.992},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.542,"5min":1.375,"15min":2.3380000000000001},"min":{"1min":0.35399999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.0830000000000002,"5min":7.0629999999999997,"15min":104.691},"last":0.69799999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":5.2640000000000002,"15min":3.552},"min":{"1min":0.44,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":1.28,"5min":183.971,"15min":183.971},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.0629999999999999,"5min":5.0650000000000004,"15min":3.351},"min":{"1min":0.40000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.9610000000000001,"5min":183.095,"15min":183.095},"last":0.59599999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.008,"15min":1.2270000000000001},"min":{"1min":0.372,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.3159999999999998,"5min":6.6239999999999997,"15min":11.029999999999999},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.296,"15min":1.5860000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.3050000000000002,"5min":15.504,"15min":15.504},"last":0.81899999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0489999999999999,"15min":1.4139999999999999},"min":{"1min":0.375,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.7130000000000001,"5min":7.1550000000000002,"15min":11.996},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.2070000000000001,"5min":0.998,"15min":1.4059999999999999},"min":{"1min":0.42399999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.6150000000000002,"5min":5.4909999999999997,"15min":13.634},"last":0.78800000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.4339999999999999,"15min":1.3240000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.2750000000000004,"5min":15.521000000000001,"15min":15.521000000000001},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.044,"5min":1.337,"15min":1.5049999999999999},"min":{"1min":0.42499999999999999,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":2.6389999999999998,"5min":15.449999999999999,"15min":15.449999999999999},"last":0.75600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626858,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299556,"kb_used_data":2332,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93445596,"statfs":{"total":95995035648,"available":95688290304,"internally_reserved":0,"allocated":2387968,"data_stored":1848964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.2929999999999999,"15min":1.373},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":32.289999999999999},"last":1.046},{"interface":"front","average":{"1min":1.982,"5min":1.355,"15min":1.4830000000000001},"min":{"1min":0.42199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.9870000000000001,"5min":6.4240000000000004,"15min":31.454999999999998},"last":0.84299999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.4339999999999999,"15min":1.819},"min":{"1min":0.42499999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":5.3760000000000003,"5min":6.9710000000000001,"15min":137.93899999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.4410000000000001,"15min":1.845},"min":{"1min":0.54400000000000004,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":4.0650000000000004,"5min":11.281000000000001,"15min":138.02500000000001},"last":0.64600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:32:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4569999999999999,"5min":2.4780000000000002,"15min":2.387},"min":{"1min":0.37,"5min":0.35699999999999998,"15min":0.32000000000000001},"max":{"1min":26.477,"5min":53.616,"15min":95.540000000000006},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.032,"5min":2.6139999999999999,"15min":2.395},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":25.745999999999999,"5min":53.665999999999997,"15min":96.352000000000004},"last":0.61099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":3.423,"15min":2.3490000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":27.491,"5min":115.09399999999999,"15min":115.09399999999999},"last":0.877},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.0329999999999999,"15min":2.0099999999999998},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":26.815000000000001,"5min":112.886,"15min":112.886},"last":0.91400000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.95799999999999996,"15min":1.2749999999999999},"min":{"1min":0.435,"5min":0.314,"15min":0.26000000000000001},"max":{"1min":1.478,"5min":6.3799999999999999,"15min":7.3090000000000002},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.071,"15min":1.4379999999999999},"min":{"1min":0.53500000000000003,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":4.1600000000000001,"5min":7.0060000000000002,"15min":10.300000000000001},"last":0.999}]},{"osd":5,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.91600000000000004,"15min":1.4510000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":1.51,"5min":6.4850000000000003,"15min":11.754},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.274,"5min":1.119,"15min":1.571},"min":{"1min":0.45200000000000001,"5min":0.38200000000000001,"15min":0.308},"max":{"1min":4.1950000000000003,"5min":12.215999999999999,"15min":12.295999999999999},"last":1.0740000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.0920000000000001,"15min":1.492},"min":{"1min":0.47199999999999998,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":3.3069999999999999,"5min":6.5110000000000001,"15min":11.076000000000001},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.0720000000000001,"15min":1.4199999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":2.8580000000000001,"5min":9.4179999999999993,"15min":11.102},"last":1.1240000000000001}]}]},{"osd":1,"up_from":93,"seq":399431958767,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":332820,"kb_used_data":2060,"kb_used_omap":7,"kb_used_meta":330744,"kb_avail":93412332,"statfs":{"total":95995035648,"available":95654227968,"internally_reserved":0,"allocated":2109440,"data_stored":1294225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":338682256},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":8.2799999999999994,"5min":4.843,"15min":2.556},"min":{"1min":0.217,"5min":0.187,"15min":0.187},"max":{"1min":132.886,"5min":132.886,"15min":132.886},"last":0.63},{"interface":"front","average":{"1min":8.5730000000000004,"5min":4.8529999999999998,"15min":2.7639999999999998},"min":{"1min":0.40500000000000003,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":133.315,"5min":133.315,"15min":133.315},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":2.637,"15min":1.9119999999999999},"min":{"1min":0.32900000000000001,"5min":0.214,"15min":0.19800000000000001},"max":{"1min":26.210000000000001,"5min":67.206000000000003,"15min":67.206000000000003},"last":0.502},{"interface":"front","average":{"1min":3.351,"5min":2.9209999999999998,"15min":1.978},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":26.026,"5min":67.132000000000005,"15min":67.132000000000005},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":8.5169999999999995,"5min":5.7839999999999998,"15min":4.0869999999999997},"min":{"1min":0.36599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":132.792,"5min":132.792,"15min":132.792},"last":0.58699999999999997},{"interface":"front","average":{"1min":8.3770000000000007,"5min":5.6539999999999999,"15min":4.048},"min":{"1min":0.26300000000000001,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":132.53299999999999,"5min":132.53299999999999,"15min":132.53299999999999},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.5840000000000001,"15min":1.5049999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":3.5939999999999999,"5min":26.068999999999999,"15min":26.068999999999999},"last":0.748},{"interface":"front","average":{"1min":1.024,"5min":1.4139999999999999,"15min":1.546},"min":{"1min":0.311,"5min":0.31,"15min":0.31},"max":{"1min":4.1790000000000003,"5min":5.6689999999999996,"15min":12.907999999999999},"last":0.86799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.5349999999999999,"15min":1.6020000000000001},"min":{"1min":0.48599999999999999,"5min":0.35899999999999999,"15min":0.32500000000000001},"max":{"1min":2.5019999999999998,"5min":10.586,"15min":14.170999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.081,"5min":1.506,"15min":1.6399999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.6150000000000002,"5min":26.157,"15min":26.157},"last":0.78900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.5880000000000001,"15min":1.524},"min":{"1min":0.52000000000000002,"5min":0.38600000000000001,"15min":0.36699999999999999},"max":{"1min":2.0790000000000002,"5min":10.625,"15min":14.212999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.224,"5min":1.6519999999999999,"15min":1.6479999999999999},"min":{"1min":0.47799999999999998,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":3.573,"5min":26.132000000000001,"15min":26.132000000000001},"last":0.95199999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.885,"15min":1.974},"min":{"1min":0.41699999999999998,"5min":0.41099999999999998,"15min":0.35399999999999998},"max":{"1min":4.4779999999999998,"5min":26.109000000000002,"15min":30.774999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.145,"5min":1.4790000000000001,"15min":1.6679999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.2290000000000001,"5min":5.8289999999999997,"15min":30.815000000000001},"last":1.004}]}]},{"osd":0,"up_from":88,"seq":377957122293,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389304,"kb_used_data":6704,"kb_used_omap":8,"kb_used_meta":382583,"kb_avail":93355848,"statfs":{"total":95995035648,"available":95596388352,"internally_reserved":0,"allocated":6864896,"data_stored":6074572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":391765248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.6760000000000002,"15min":2.613},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":12.991,"5min":13.598000000000001,"15min":138.31700000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.2429999999999999,"15min":2.3759999999999999},"min":{"1min":0.28100000000000003,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":9.7319999999999993,"5min":12.101000000000001,"15min":144.37},"last":0.81399999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:32:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7890000000000001,"5min":2.5019999999999998,"15min":1.958},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":28.745000000000001},"last":0.621},{"interface":"front","average":{"1min":3.2599999999999998,"5min":2.0510000000000002,"15min":1.9199999999999999},"min":{"1min":0.22600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":16.129000000000001,"5min":16.129000000000001,"15min":28.146999999999998},"last":0.64700000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":3.1150000000000002,"15min":2.3580000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.20899999999999999},"max":{"1min":5.3289999999999997,"5min":56,"15min":59.643000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.82899999999999996,"5min":3.032,"15min":2.1880000000000002},"min":{"1min":0.38900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9690000000000001,"5min":54.524000000000001,"15min":59.762999999999998},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:33:07 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":1.6399999999999999,"15min":1.585},"min":{"1min":0.42899999999999999,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":5.1920000000000002,"5min":8.9920000000000009,"15min":9.0890000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.6890000000000001,"5min":1.8959999999999999,"15min":1.851},"min":{"1min":0.623,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.7439999999999998,"5min":9.2319999999999993,"15min":28.629000000000001},"last":0.94299999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:33:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.0049999999999999,"15min":2.0739999999999998},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.35399999999999998},"max":{"1min":11.042999999999999,"5min":12.356999999999999,"15min":28.574000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.905,"15min":1.8220000000000001},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.32600000000000001},"max":{"1min":4.9379999999999997,"5min":9.8209999999999997,"15min":28.061},"last":0.88800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:33:07 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.885,"15min":1.7370000000000001},"min":{"1min":0.47799999999999998,"5min":0.35299999999999998,"15min":0.34599999999999997},"max":{"1min":4.9009999999999998,"5min":16.181999999999999,"15min":27.966999999999999},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.643,"15min":1.494},"min":{"1min":0.48999999999999999,"5min":0.39300000000000002,"15min":0.33700000000000002},"max":{"1min":5.2889999999999997,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":0.87}]},{"osd":7,"last update":"Mon Oct 9 06:33:07 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.8340000000000001,"15min":1.8340000000000001},"min":{"1min":0.47899999999999998,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":4.8579999999999997,"5min":7.9169999999999998,"15min":28.109999999999999},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.6890000000000001,"5min":1.8759999999999999,"15min":1.8859999999999999},"min":{"1min":0.46999999999999997,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":6.7110000000000003,"5min":8.9190000000000005,"15min":28.074999999999999},"last":1.0760000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795242,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408636,"kb_used_data":6132,"kb_used_omap":11,"kb_used_meta":402484,"kb_avail":93336516,"statfs":{"total":95995035648,"available":95576592384,"internally_reserved":0,"allocated":6279168,"data_stored":5482698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":412144574},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":3.0790000000000002,"15min":2.2869999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.17899999999999999},"max":{"1min":15.301,"5min":117.2,"15min":117.2},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.744,"5min":2.7679999999999998,"15min":2.3740000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":8.7449999999999992,"5min":121.521,"15min":121.521},"last":0.57699999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.5880000000000001,"15min":1.6419999999999999},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":4.3799999999999999,"5min":12.737,"15min":12.737},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.4379999999999999,"15min":1.8420000000000001},"min":{"1min":0.26100000000000001,"5min":0.215,"15min":0.215},"max":{"1min":4.0439999999999996,"5min":10.997999999999999,"15min":11.943},"last":0.47399999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:33:18 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":5.8869999999999996,"15min":3.3439999999999999},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":6.4370000000000003,"5min":201.33600000000001,"15min":201.33600000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.0640000000000001,"5min":5.5549999999999997,"15min":3.2829999999999999},"min":{"1min":0.40500000000000003,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":3.0510000000000002,"5min":199.95099999999999,"15min":199.95099999999999},"last":0.60099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.3939999999999999,"15min":2.0059999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.339,"5min":7.5759999999999996,"15min":19.184000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.21,"15min":1.786},"min":{"1min":0.437,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":4.5819999999999999,"5min":6.3049999999999997,"15min":29.408000000000001},"last":0.90000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.3570000000000002,"15min":2.3519999999999999},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.31},"max":{"1min":3.2759999999999998,"5min":86.423000000000002,"15min":86.423000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":2.085,"5min":2.23,"15min":2.0129999999999999},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":12.532999999999999,"5min":86.382999999999996,"15min":86.382999999999996},"last":0.85099999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":1.484,"15min":1.641},"min":{"1min":0.50800000000000001,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":8.2959999999999994,"5min":8.2959999999999994,"15min":29.504000000000001},"last":0.747},{"interface":"front","average":{"1min":1.879,"5min":1.4470000000000001,"15min":1.673},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":6.343,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2999999999999998,"5min":1.591,"15min":1.7969999999999999},"min":{"1min":0.42599999999999999,"5min":0.41599999999999998,"15min":0.308},"max":{"1min":9.2050000000000001,"5min":9.2050000000000001,"15min":9.2050000000000001},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.579,"5min":1.5860000000000001,"15min":2.0129999999999999},"min":{"1min":0.39300000000000002,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":4.2380000000000004,"5min":7.6920000000000002,"15min":12.051},"last":0.73599999999999999}]}]},{"osd":3,"up_from":103,"seq":442381631716,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":426752,"kb_used_data":6136,"kb_used_omap":3,"kb_used_meta":420604,"kb_avail":93318400,"statfs":{"total":95995035648,"available":95558041600,"internally_reserved":0,"allocated":6283264,"data_stored":5482919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":430699041},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":2.411,"15min":2.593},"min":{"1min":0.28299999999999997,"5min":0.184,"15min":0.16900000000000001},"max":{"1min":12.952,"5min":15.285,"15min":147.988},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.1720000000000002,"15min":2.4380000000000002},"min":{"1min":0.23599999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":23.646000000000001,"5min":23.646000000000001,"15min":147.94200000000001},"last":0.443}]},{"osd":1,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.4790000000000001,"15min":1.6970000000000001},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.19},"max":{"1min":8.673,"5min":47.057000000000002,"15min":47.057000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.887,"5min":2.968,"15min":2.0390000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.218},"max":{"1min":10.488,"5min":54.752000000000002,"15min":54.752000000000002},"last":0.54300000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":3.0110000000000001,"15min":2.504},"min":{"1min":0.29899999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.968,"5min":47.097999999999999,"15min":70.594999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.3169999999999999,"5min":2.4929999999999999,"15min":2.0339999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":4.0730000000000004,"5min":46.997,"15min":62.991999999999997},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":2.641,"15min":1.964},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6470000000000002,"5min":72.653000000000006,"15min":72.653000000000006},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.444,"15min":2.0459999999999998},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":3.1779999999999999,"5min":72.706999999999994,"15min":72.706999999999994},"last":0.65000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.8440000000000001,"15min":1.7390000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":3.1459999999999999,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.1200000000000001,"5min":1.962,"15min":1.8180000000000001},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.6739999999999995,"5min":9.4619999999999997,"15min":11.754},"last":0.70499999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.98,"15min":1.6890000000000001},"min":{"1min":0.55500000000000005,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":10.384,"5min":15.122999999999999,"15min":15.122999999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.9670000000000001,"15min":1.649},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.2080000000000002,"5min":11.728999999999999,"15min":11.728999999999999},"last":0.91500000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.0870000000000002,"15min":1.8300000000000001},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":18.695},"last":0.747},{"interface":"front","average":{"1min":1.7769999999999999,"5min":2.1240000000000001,"15min":1.9259999999999999},"min":{"1min":0.52200000000000002,"5min":0.43099999999999999,"15min":0.39900000000000002},"max":{"1min":10.409000000000001,"5min":13.211,"15min":13.211},"last":0.87}]}]},{"osd":4,"up_from":30,"seq":128849019324,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307180,"kb_used_data":1764,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437972,"statfs":{"total":95995035648,"available":95680483328,"internally_reserved":0,"allocated":1806336,"data_stored":1259283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":2.488,"15min":2.452},"min":{"1min":0.28999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":16.218,"5min":115.59699999999999,"15min":115.59699999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.6659999999999999,"5min":2.6499999999999999,"15min":2.778},"min":{"1min":0.33500000000000002,"5min":0.28000000000000003,"15min":0.27700000000000002},"max":{"1min":6.4880000000000004,"5min":115.65000000000001,"15min":115.65000000000001},"last":0.78200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.292,"15min":2.2130000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":11.846,"5min":11.846,"15min":179.47300000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.603,"5min":1.476,"15min":2.6360000000000001},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.28499999999999998},"max":{"1min":5.1619999999999999,"5min":5.3360000000000003,"15min":179.447},"last":0.84099999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:32:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.6319999999999999,"15min":2.153},"min":{"1min":0.38300000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.705,"5min":14.705,"15min":61.783999999999999},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.6020000000000001,"15min":2.29},"min":{"1min":0.35399999999999998,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.7850000000000001,"5min":9.7759999999999998,"15min":61.551000000000002},"last":0.55400000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":2.1709999999999998,"15min":2.4199999999999999},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":5.548,"5min":12.081,"15min":91.361999999999995},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.9550000000000001,"15min":2.0179999999999998},"min":{"1min":0.40799999999999997,"5min":0.36699999999999999,"15min":0.313},"max":{"1min":16.257000000000001,"5min":16.257000000000001,"15min":89.676000000000002},"last":0.97999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:32:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":0.83999999999999997,"15min":1.298},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.737,"5min":11.737,"15min":12.242000000000001},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.24,"5min":1.0680000000000001,"15min":1.6899999999999999},"min":{"1min":0.47799999999999998,"5min":0.36199999999999999,"15min":0.26500000000000001},"max":{"1min":4.5060000000000002,"5min":4.5060000000000002,"15min":20.113},"last":0.89000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.337,"15min":1.5920000000000001},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.29499999999999998},"max":{"1min":6.5990000000000002,"5min":8.1110000000000007,"15min":18.010000000000002},"last":1.157},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.268,"15min":1.4350000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.782,"5min":11.782,"15min":13.913},"last":1.238}]},{"osd":7,"last update":"Mon Oct 9 06:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.464,"15min":1.7749999999999999},"min":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.371},"max":{"1min":6.8310000000000004,"5min":12.779,"15min":15.294},"last":1.21},{"interface":"front","average":{"1min":1.734,"5min":1.2250000000000001,"15min":1.7569999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.1370000000000005,"5min":9.1370000000000005,"15min":14.896000000000001},"last":1.0249999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823091,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315352,"kb_used_data":1744,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":93429800,"statfs":{"total":95995035648,"available":95672115200,"internally_reserved":0,"allocated":1785856,"data_stored":1254416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:50 2023","interfaces":[{"interface":"back","average":{"1min":5.0469999999999997,"5min":2.9220000000000002,"15min":2.7589999999999999},"min":{"1min":0.26700000000000002,"5min":0.254,"15min":0.247},"max":{"1min":83.197000000000003,"5min":83.197000000000003,"15min":83.197000000000003},"last":0.628},{"interface":"front","average":{"1min":5.4960000000000004,"5min":2.7639999999999998,"15min":2.4710000000000001},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":83.563999999999993,"5min":83.563999999999993,"15min":83.563999999999993},"last":0.67800000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.823,"5min":1.3300000000000001,"15min":2.3940000000000001},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.4779999999999998,"5min":5.9299999999999997,"15min":161.71199999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.397,"5min":1.516,"15min":2.6480000000000001},"min":{"1min":0.33600000000000002,"5min":0.31,"15min":0.29399999999999998},"max":{"1min":4.806,"5min":9.0589999999999993,"15min":162.50200000000001},"last":0.72799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:32:43 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":2.1160000000000001,"15min":2.2839999999999998},"min":{"1min":0.34999999999999998,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":5.1050000000000004,"5min":54.981999999999999,"15min":54.981999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.9159999999999999,"5min":2.1749999999999998,"15min":2.2570000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":9.8889999999999993,"5min":55.008000000000003,"15min":59.082999999999998},"last":0.82099999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":5.3209999999999997,"5min":4.1379999999999999,"15min":2.8740000000000001},"min":{"1min":0.438,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":84.174000000000007,"5min":84.174000000000007,"15min":84.174000000000007},"last":0.93500000000000005},{"interface":"front","average":{"1min":6.0369999999999999,"5min":4.1120000000000001,"15min":2.7120000000000002},"min":{"1min":0.41399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":83.114999999999995,"5min":83.114999999999995,"15min":83.114999999999995},"last":0.92100000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:32:31 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.208,"15min":1.879},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1980000000000004,"5min":8.8879999999999999,"15min":17.667999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.117,"15min":1.599},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3040000000000003,"5min":5.3040000000000003,"15min":16.181999999999999},"last":1.109}]},{"osd":6,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.2609999999999999,"15min":1.702},"min":{"1min":0.442,"5min":0.37,"15min":0.35699999999999998},"max":{"1min":3.5699999999999998,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.008,"5min":1.264,"15min":1.6160000000000001},"min":{"1min":0.47799999999999998,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":1.9350000000000001,"5min":9.3450000000000006,"15min":12.365},"last":1.1819999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.0840000000000001,"15min":1.702},"min":{"1min":0.47599999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.7039999999999997,"5min":5.7039999999999997,"15min":13.047000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.214,"15min":1.5349999999999999},"min":{"1min":0.42499999999999999,"5min":0.41599999999999998,"15min":0.36799999999999999},"max":{"1min":5.5919999999999996,"5min":5.5919999999999996,"15min":11.981},"last":1.141}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:33:36.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:33:36.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:36.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:36.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:36.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:36.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:36.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:37.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:37.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:37.244 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:37.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:37.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:37.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:37.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:37.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:37.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:37.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:37.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:37.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:37.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:37.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:38.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:38.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:38.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:38.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:38.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:38 smithi125 bash[80002]: cluster 2023-10-09T06:33:37.108757+0000 mgr.y (mgr.25723) 864 : cluster 0 pgmap v872: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:38.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:38 smithi125 bash[77299]: cluster 2023-10-09T06:33:37.108757+0000 mgr.y (mgr.25723) 864 : cluster 0 pgmap v872: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:38.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:38.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:38.431 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:38 smithi155 bash[59495]: cluster 2023-10-09T06:33:37.108757+0000 mgr.y (mgr.25723) 864 : cluster 0 pgmap v872: 89 pgs: 4 creating+peering, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:38.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:38.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:38.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:38.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:38.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:38.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:38.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:39.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:39.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:39.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:39.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:39.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:39.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:39.325 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:33:39.351 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:33:39.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:39.377 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:33:39.405 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:33:39.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:39.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:39.434 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:33:39.458 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:33:39.485 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:33:39.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:39.514 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:33:39.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:39.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:39.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:39.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:39.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:39.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:39.912 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:33:39.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:39.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:39.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:40.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:40.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[77299]: cluster 2023-10-09T06:33:39.109396+0000 mgr.y (mgr.25723) 865 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[77299]: audit 2023-10-09T06:33:39.292717+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.125:0/2677301568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[77299]: audit 2023-10-09T06:33:39.472298+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.125:0/2417118212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[77299]: audit 2023-10-09T06:33:39.472729+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[80002]: cluster 2023-10-09T06:33:39.109396+0000 mgr.y (mgr.25723) 865 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[80002]: audit 2023-10-09T06:33:39.292717+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.125:0/2677301568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[80002]: audit 2023-10-09T06:33:39.472298+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.125:0/2417118212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:39 smithi125 bash[80002]: audit 2023-10-09T06:33:39.472729+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.200 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:39 smithi155 bash[59495]: cluster 2023-10-09T06:33:39.109396+0000 mgr.y (mgr.25723) 865 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:40.200 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:39 smithi155 bash[59495]: audit 2023-10-09T06:33:39.292717+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.125:0/2677301568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:40.200 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:39 smithi155 bash[59495]: audit 2023-10-09T06:33:39.472298+0000 mon.c (mon.2) 85 : audit 1 from='client.? 172.21.15.125:0/2417118212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.200 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:39 smithi155 bash[59495]: audit 2023-10-09T06:33:39.472729+0000 mon.a (mon.0) 1074 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:40.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:40.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:40.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:40.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:40.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:40.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:40.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:40.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:40.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:40.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:41.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:41.031 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:41.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:40 smithi155 bash[59495]: cluster 2023-10-09T06:33:39.889300+0000 mon.a (mon.0) 1075 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:40 smithi155 bash[59495]: audit 2023-10-09T06:33:39.901396+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:40 smithi155 bash[59495]: cluster 2023-10-09T06:33:39.901474+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:33:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:40 smithi155 bash[59495]: cluster 2023-10-09T06:33:39.902922+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:40 smithi155 bash[59495]: cluster 2023-10-09T06:33:39.902925+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:41.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:41.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:41] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.33.4" 2023-10-09T06:33:41.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[80002]: cluster 2023-10-09T06:33:39.889300+0000 mon.a (mon.0) 1075 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:41.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[80002]: audit 2023-10-09T06:33:39.901396+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:41.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[80002]: cluster 2023-10-09T06:33:39.901474+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:33:41.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[80002]: cluster 2023-10-09T06:33:39.902922+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[80002]: cluster 2023-10-09T06:33:39.902925+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[77299]: cluster 2023-10-09T06:33:39.889300+0000 mon.a (mon.0) 1075 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:41.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[77299]: audit 2023-10-09T06:33:39.901396+0000 mon.a (mon.0) 1076 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:41.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[77299]: cluster 2023-10-09T06:33:39.901474+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:33:41.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[77299]: cluster 2023-10-09T06:33:39.902922+0000 osd.6 (osd.6) 132 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:40 smithi125 bash[77299]: cluster 2023-10-09T06:33:39.902925+0000 osd.4 (osd.4) 124 : cluster 3 failed to encode map e195 with expected crc 2023-10-09T06:33:41.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:41.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:41.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:41.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:41.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:41.710 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:41.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.832 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:41.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:41.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:41.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:41.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:42.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:42.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:41 smithi155 bash[59495]: cluster 2023-10-09T06:33:41.110105+0000 mgr.y (mgr.25723) 866 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:42.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:42.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:41 smithi125 bash[77299]: cluster 2023-10-09T06:33:41.110105+0000 mgr.y (mgr.25723) 866 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:42.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:41 smithi125 bash[80002]: cluster 2023-10-09T06:33:41.110105+0000 mgr.y (mgr.25723) 866 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:42.202 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:42.202 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:42.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:42.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:42.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:42.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:42.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:42.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:42.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:42.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:42.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:33:42.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:42.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:42.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:42.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:42.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:42.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:43.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:43.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:43.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:43.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:43.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:33:43.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:43.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:43.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:43.679 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:43.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:43.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:43.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:43.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:43.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:44.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:44.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:44.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:44.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:44.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:44.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:44 smithi155 bash[59495]: cluster 2023-10-09T06:33:43.110698+0000 mgr.y (mgr.25723) 867 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:44.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:44 smithi155 bash[59495]: audit 2023-10-09T06:33:44.057203+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:44.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:44 smithi125 bash[80002]: cluster 2023-10-09T06:33:43.110698+0000 mgr.y (mgr.25723) 867 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:44 smithi125 bash[80002]: audit 2023-10-09T06:33:44.057203+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:44 smithi125 bash[77299]: cluster 2023-10-09T06:33:43.110698+0000 mgr.y (mgr.25723) 867 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:44 smithi125 bash[77299]: audit 2023-10-09T06:33:44.057203+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:44.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:44.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:44.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:44.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:44.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:44.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:44.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:44.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:44.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:44.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:45.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:45.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:45.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:45.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:45.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:45.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:45.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:45.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:45.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:45.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:45.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:45.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:45.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:45.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:45.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:45.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:45.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:46.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:46.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:46.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:46.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:46.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:46.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:46 smithi155 bash[59495]: cluster 2023-10-09T06:33:45.111340+0000 mgr.y (mgr.25723) 868 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:46.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:46.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:46.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:46 smithi125 bash[77299]: cluster 2023-10-09T06:33:45.111340+0000 mgr.y (mgr.25723) 868 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:46.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:46 smithi125 bash[80002]: cluster 2023-10-09T06:33:45.111340+0000 mgr.y (mgr.25723) 868 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:46.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:46.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:46.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:46.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:46.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:46.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:46.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:47.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:47.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:47.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:47.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:47.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:47.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:47.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:47.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:47.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:47.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:47.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:47.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:47.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:47.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:48.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:48.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:48.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:48.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:48.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:48.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:48.258 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:48 smithi125 bash[77299]: cluster 2023-10-09T06:33:47.111848+0000 mgr.y (mgr.25723) 869 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:48.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:48 smithi125 bash[80002]: cluster 2023-10-09T06:33:47.111848+0000 mgr.y (mgr.25723) 869 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:48.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:48.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:48.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:48 smithi155 bash[59495]: cluster 2023-10-09T06:33:47.111848+0000 mgr.y (mgr.25723) 869 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:48.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:48.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:48.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:48.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:48.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:48.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:48.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:48.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:48.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:48.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:49.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:49.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:49.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:49.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:49.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:49.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:49.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:49.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:49.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:49.589 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:49.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:49.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[77299]: audit 2023-10-09T06:33:48.594530+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:49.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[77299]: audit 2023-10-09T06:33:48.595793+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:49.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[77299]: cluster 2023-10-09T06:33:49.112625+0000 mgr.y (mgr.25723) 870 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:49.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[80002]: audit 2023-10-09T06:33:48.594530+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:49.706 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[80002]: audit 2023-10-09T06:33:48.595793+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:49.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:49 smithi125 bash[80002]: cluster 2023-10-09T06:33:49.112625+0000 mgr.y (mgr.25723) 870 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:49.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:49.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:49.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:49 smithi155 bash[59495]: audit 2023-10-09T06:33:48.594530+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:33:49.834 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:49 smithi155 bash[59495]: audit 2023-10-09T06:33:48.595793+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:49.835 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:49 smithi155 bash[59495]: cluster 2023-10-09T06:33:49.112625+0000 mgr.y (mgr.25723) 870 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:49.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:49.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:49.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:49.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:50.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:50.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:50.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:50.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:50.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:50.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:50.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:50.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:50.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:50.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:50.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:50.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:50.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:50.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:50.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:50.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:51.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:51.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:51.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:51.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:51.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:51.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:51.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:51.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:51.451 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:33:51.451 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:33:51.451 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T06:33:51.451 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:33:51.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:51.477 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:33:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:33:51] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T06:33:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:51.552 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:51.553 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:51.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:51.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:51.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:51.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:51.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:51.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:51.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:51.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:51.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:52.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:52.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:52.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:52.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:52.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:52.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:52.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:52.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:52.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:52 smithi155 bash[59495]: cluster 2023-10-09T06:33:51.113568+0000 mgr.y (mgr.25723) 871 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:52.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:52.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:52.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:52 smithi125 bash[77299]: cluster 2023-10-09T06:33:51.113568+0000 mgr.y (mgr.25723) 871 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:52.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:52 smithi125 bash[80002]: cluster 2023-10-09T06:33:51.113568+0000 mgr.y (mgr.25723) 871 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:52.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:52.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:52.825 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:33:52.825 INFO:teuthology.orchestra.run.smithi125.stdout:{"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1346,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":195,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832081,"num_in_osds":8,"osd_in_since":1696832909,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":28,"data_bytes":4790195,"bytes_used":2850381824,"bytes_avail":765109903360,"bytes_total":767960285184},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi125.yetyhy","status":"up:active","gid":24542}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.125:8443/","prometheus":"http://172.21.15.125:9283/"}},"servicemap":{"epoch":30,"modified":"2023-10-09T06:30:41.044890+0000","services":{}},"progress_events":{"74b9a7a6-3b80-432e-a0ed-33973b35b600":{"message":"Upgrade to 18.2.0-598-gb4fa663f (4m)\n [===========.................] (remaining: 7m)","progress":0.40000000596046448,"add_to_ceph_s":true}}} 2023-10-09T06:33:52.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:52.893 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:52.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:52.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:53.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:53.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:53.124 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:53.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:53.192 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T06:33:53.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:53.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:53.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:53 smithi155 bash[59495]: audit 2023-10-09T06:33:52.818329+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.125:0/501362003' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:33:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:53 smithi155 bash[59495]: audit 2023-10-09T06:33:53.116895+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.125:0/4110499013' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:53 smithi155 bash[59495]: audit 2023-10-09T06:33:53.117235+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:53.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[80002]: audit 2023-10-09T06:33:52.818329+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.125:0/501362003' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[80002]: audit 2023-10-09T06:33:53.116895+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.125:0/4110499013' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[80002]: audit 2023-10-09T06:33:53.117235+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[77299]: audit 2023-10-09T06:33:52.818329+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.125:0/501362003' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[77299]: audit 2023-10-09T06:33:53.116895+0000 mon.c (mon.2) 87 : audit 1 from='client.? 172.21.15.125:0/4110499013' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.487 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:53 smithi125 bash[77299]: audit 2023-10-09T06:33:53.117235+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:33:53.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:53.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:53.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:53.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:53.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:53.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:53.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:53.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:53.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:53.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:54.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:54.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:54.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:54.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:54.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:54.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[80002]: cluster 2023-10-09T06:33:53.114339+0000 mgr.y (mgr.25723) 872 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[80002]: audit 2023-10-09T06:33:53.182097+0000 mon.a (mon.0) 1082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[80002]: cluster 2023-10-09T06:33:53.182249+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[80002]: cluster 2023-10-09T06:33:53.182282+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[77299]: cluster 2023-10-09T06:33:53.114339+0000 mgr.y (mgr.25723) 872 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[77299]: audit 2023-10-09T06:33:53.182097+0000 mon.a (mon.0) 1082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[77299]: cluster 2023-10-09T06:33:53.182249+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T06:33:54.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:54 smithi125 bash[77299]: cluster 2023-10-09T06:33:53.182282+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:33:54.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:54.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:54.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:54.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:54.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:54 smithi155 bash[59495]: cluster 2023-10-09T06:33:53.114339+0000 mgr.y (mgr.25723) 872 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:54.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:54 smithi155 bash[59495]: audit 2023-10-09T06:33:53.182097+0000 mon.a (mon.0) 1082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:33:54.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:54 smithi155 bash[59495]: cluster 2023-10-09T06:33:53.182249+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e196 with expected crc 2023-10-09T06:33:54.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:54 smithi155 bash[59495]: cluster 2023-10-09T06:33:53.182282+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:33:54.603 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:54.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:54.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:54.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:54.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:54.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:54.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:55.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:55.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:55.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:55.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:55.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:55.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:55.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:55.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:33:55.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:55.737 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T06:33:55.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:33:55.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:55.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:55.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:55.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:56.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:56.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:56.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:56.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:56.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:56.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:56 smithi125 bash[80002]: cluster 2023-10-09T06:33:55.115466+0000 mgr.y (mgr.25723) 873 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:56.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:56 smithi125 bash[77299]: cluster 2023-10-09T06:33:55.115466+0000 mgr.y (mgr.25723) 873 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:56.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:56.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:56.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:56 smithi155 bash[59495]: cluster 2023-10-09T06:33:55.115466+0000 mgr.y (mgr.25723) 873 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:56.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:56.737 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T06:33:56.739 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:33:56.740 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:33:56.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:56.779 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:33:56.779 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:33:56.779 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_176369 2023-10-09T06:33:56.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:33:56.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:33:56.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:56.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:56.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:56.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:57.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:57.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:57.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:57.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:57.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:57.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:57.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:57.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:57.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:57.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:57.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:57.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:57.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2732 2716 169.741 169.75 0.00349714 0.00586413 2023-10-09T06:33:57.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:57.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:57.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:57.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:57.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:58.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:58.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:58.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:58.192 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:58 smithi125 bash[77299]: cluster 2023-10-09T06:33:57.116726+0000 mgr.y (mgr.25723) 874 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:58.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:58.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:58.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:58.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:58 smithi155 bash[59495]: cluster 2023-10-09T06:33:57.116726+0000 mgr.y (mgr.25723) 874 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:58.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:58.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:58.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:58.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:58 smithi125 bash[80002]: cluster 2023-10-09T06:33:57.116726+0000 mgr.y (mgr.25723) 874 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:33:58.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:58.705 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:58.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:58.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 5549 5533 172.893 176.062 0.0109732 0.0057699 2023-10-09T06:33:58.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:58.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:58.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:58.990 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:58.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:58.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:59.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:59.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:59.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:59.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:59.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:59.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:59.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:59.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:59.621 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:59.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:59.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:59.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 8198 8182 170.444 165.562 0.0116487 0.00585731 2023-10-09T06:33:59.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:59.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:59.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:33:59.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:00.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:00.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:00.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:00.170 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:33:59 smithi155 bash[59495]: cluster 2023-10-09T06:33:59.117282+0000 mgr.y (mgr.25723) 875 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 92 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 140 op/s 2023-10-09T06:34:00.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:00.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:00.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:00.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:33:59 smithi125 bash[77299]: cluster 2023-10-09T06:33:59.117282+0000 mgr.y (mgr.25723) 875 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 92 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 140 op/s 2023-10-09T06:34:00.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:33:59 smithi125 bash[80002]: cluster 2023-10-09T06:33:59.117282+0000 mgr.y (mgr.25723) 875 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 92 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.8 MiB/s wr, 140 op/s 2023-10-09T06:34:00.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:00.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:00.476 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:00.476 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:00.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:00.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:00.693 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:00.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:00.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 10640 10624 165.986 152.625 0.00788298 0.00601613 2023-10-09T06:34:00.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:00.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:00.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:00.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:00.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:00.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:01.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:01.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:01.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:01.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:01.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:01.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:01] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.33.4" 2023-10-09T06:34:01.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:01.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:01.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:01.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:01.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:01.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:01.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:01.780 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 13056 13040 162.986 151 0.00495997 0.00612618 2023-10-09T06:34:01.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:01.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:01.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:01.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:02.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:02.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:02.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:02.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:02.293 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:02 smithi155 bash[59495]: cluster 2023-10-09T06:34:01.117981+0000 mgr.y (mgr.25723) 876 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 282 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s wr, 444 op/s 2023-10-09T06:34:02.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:02.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:02.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:02.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:02.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:02 smithi125 bash[77299]: cluster 2023-10-09T06:34:01.117981+0000 mgr.y (mgr.25723) 876 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 282 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s wr, 444 op/s 2023-10-09T06:34:02.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:02 smithi125 bash[80002]: cluster 2023-10-09T06:34:01.117981+0000 mgr.y (mgr.25723) 876 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 282 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s wr, 444 op/s 2023-10-09T06:34:02.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:02.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:02.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 15042 15026 156.507 124.125 0.00491788 0.00638334 2023-10-09T06:34:02.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:02.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:02.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:02.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:03.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:03.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:03.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:03.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:03.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:03.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:03.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:03.651 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:03.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:03.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 17281 17265 154.138 139.938 0.00188362 0.00643943 2023-10-09T06:34:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:03.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:03.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:03.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:04.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:04.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:04.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:04.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:04.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:04 smithi155 bash[59495]: cluster 2023-10-09T06:34:03.118614+0000 mgr.y (mgr.25723) 877 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 629 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T06:34:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:04 smithi155 bash[59495]: audit 2023-10-09T06:34:03.588517+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:04.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:04.419 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:04 smithi125 bash[77299]: cluster 2023-10-09T06:34:03.118614+0000 mgr.y (mgr.25723) 877 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 629 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T06:34:04.419 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:04 smithi125 bash[77299]: audit 2023-10-09T06:34:03.588517+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:04.419 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:04 smithi125 bash[80002]: cluster 2023-10-09T06:34:03.118614+0000 mgr.y (mgr.25723) 877 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 629 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2023-10-09T06:34:04.420 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:04 smithi125 bash[80002]: audit 2023-10-09T06:34:03.588517+0000 mon.a (mon.0) 1084 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:04.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:04.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:04.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:04.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 18783 18767 146.604 93.875 0.00807683 0.00681135 2023-10-09T06:34:04.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:04.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:04.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:04.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:04.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:05.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:05.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:05.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:05.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:05.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:05.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:05.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:05.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:05.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:05.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:05.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:05.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:05.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:05.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 20698 20682 143.612 119.688 0.00651264 0.00695654 2023-10-09T06:34:05.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:05.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:05.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:05.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:06.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:06.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:06.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:06.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:06.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:06.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:06.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:06 smithi125 bash[77299]: cluster 2023-10-09T06:34:05.119345+0000 mgr.y (mgr.25723) 878 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:34:06.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:06 smithi125 bash[80002]: cluster 2023-10-09T06:34:05.119345+0000 mgr.y (mgr.25723) 878 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:34:06.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:06 smithi155 bash[59495]: cluster 2023-10-09T06:34:05.119345+0000 mgr.y (mgr.25723) 878 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 1020 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:34:06.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:06.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:06.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:06.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:06.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:06.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:06.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 22730 22714 141.95 127 0.00492136 0.00703887 2023-10-09T06:34:06.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:06.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:06.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:06.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:06.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:06.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:07.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:07.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:07.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:07.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:07.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:07.282 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:34:07.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:07.328 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:34:07.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:07.365 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:34:07.402 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:34:07.442 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:34:07.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:07.483 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:34:07.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:07.523 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:34:07.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.561 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:34:07.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:07.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:07.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 24263 24247 137.755 95.8125 0.0085042 0.00725237 2023-10-09T06:34:07.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:07.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:07.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:08.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:08.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:08.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:08.225 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:34:08.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:08.258 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[80002]: cluster 2023-10-09T06:34:07.119946+0000 mgr.y (mgr.25723) 879 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:34:08.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[80002]: audit 2023-10-09T06:34:07.210310+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.125:0/3520149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:08.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[80002]: audit 2023-10-09T06:34:07.504846+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.125:0/3618062264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[80002]: audit 2023-10-09T06:34:07.506451+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[77299]: cluster 2023-10-09T06:34:07.119946+0000 mgr.y (mgr.25723) 879 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:34:08.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[77299]: audit 2023-10-09T06:34:07.210310+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.125:0/3520149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:08.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[77299]: audit 2023-10-09T06:34:07.504846+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.125:0/3618062264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.260 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:08 smithi125 bash[77299]: audit 2023-10-09T06:34:07.506451+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:08.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:08.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:08.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:08.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:08.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:08 smithi155 bash[59495]: cluster 2023-10-09T06:34:07.119946+0000 mgr.y (mgr.25723) 879 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:34:08.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:08 smithi155 bash[59495]: audit 2023-10-09T06:34:07.210310+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.125:0/3520149563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:08.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:08 smithi155 bash[59495]: audit 2023-10-09T06:34:07.504846+0000 mon.c (mon.2) 89 : audit 1 from='client.? 172.21.15.125:0/3618062264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:08 smithi155 bash[59495]: audit 2023-10-09T06:34:07.506451+0000 mon.a (mon.0) 1085 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:08.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:08.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:08.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:08.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:08.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 26151 26135 136.108 118 0.00803137 0.00734059 2023-10-09T06:34:08.784 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:08.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:08.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:08.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:08.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:08.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:08.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:09.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:09.099 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:09.203 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:09.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:09.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:09.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:09.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:09 smithi155 bash[59495]: cluster 2023-10-09T06:34:08.187882+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:09 smithi155 bash[59495]: audit 2023-10-09T06:34:08.210152+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:09.473 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:09 smithi155 bash[59495]: cluster 2023-10-09T06:34:08.210365+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:34:09.474 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:09 smithi155 bash[59495]: cluster 2023-10-09T06:34:08.212010+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T06:34:09.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:09.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:09.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[77299]: cluster 2023-10-09T06:34:08.187882+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[77299]: audit 2023-10-09T06:34:08.210152+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[77299]: cluster 2023-10-09T06:34:08.210365+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[77299]: cluster 2023-10-09T06:34:08.212010+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[80002]: cluster 2023-10-09T06:34:08.187882+0000 mon.a (mon.0) 1086 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[80002]: audit 2023-10-09T06:34:08.210152+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[80002]: cluster 2023-10-09T06:34:08.210365+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:34:09.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:09 smithi125 bash[80002]: cluster 2023-10-09T06:34:08.212010+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e197 with expected crc 2023-10-09T06:34:09.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:09.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:09.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:09.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 28083 28067 134.926 120.75 0.00284278 0.00740445 2023-10-09T06:34:09.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:09.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:09.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:10.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:10.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:10.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:10.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:10.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:10.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:10.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:10.278 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:10 smithi155 bash[59495]: cluster 2023-10-09T06:34:09.120372+0000 mgr.y (mgr.25723) 880 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:34:10.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:10.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:10.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:10.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:10.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:10.539 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:10 smithi125 bash[77299]: cluster 2023-10-09T06:34:09.120372+0000 mgr.y (mgr.25723) 880 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:34:10.540 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:10 smithi125 bash[80002]: cluster 2023-10-09T06:34:09.120372+0000 mgr.y (mgr.25723) 880 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:34:10.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:10.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:10.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:10.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 29889 29873 133.35 112.875 0.00922707 0.0074936 2023-10-09T06:34:10.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:10.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:10.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:10.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:10.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:11.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:11.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:11.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:11.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:11.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:11.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:11.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:11.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.482 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:11] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-10-09T06:34:11.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:11.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:11.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:11.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:11.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:11.781 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 31568 31552 131.455 104.938 0.0041225 0.00760076 2023-10-09T06:34:11.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:11.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:11.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:11.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:11.969 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:12.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:12.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:12.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:12.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:12.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:12.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:12 smithi155 bash[59495]: cluster 2023-10-09T06:34:11.120915+0000 mgr.y (mgr.25723) 881 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:34:12.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:12.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:12.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:12 smithi125 bash[77299]: cluster 2023-10-09T06:34:11.120915+0000 mgr.y (mgr.25723) 881 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:34:12.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:12 smithi125 bash[80002]: cluster 2023-10-09T06:34:11.120915+0000 mgr.y (mgr.25723) 881 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:34:12.414 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:12.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:12.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:12.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:12.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:12.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:12.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:12.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 33141 33125 129.383 98.3125 0.00778766 0.00772303 2023-10-09T06:34:12.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:12.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:12.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:12.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:12.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:13.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:13.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:13.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:13.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:13.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:13.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:13.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:13.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:13.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:13.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:13.782 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 34843 34827 128.029 106.375 0.0073001 0.00780511 2023-10-09T06:34:13.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:13.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:13.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:14.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:14.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:14.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:14.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:14.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:14.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:14.389 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:14 smithi155 bash[59495]: cluster 2023-10-09T06:34:13.121482+0000 mgr.y (mgr.25723) 882 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T06:34:14.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:14.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:14.517 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:14 smithi125 bash[77299]: cluster 2023-10-09T06:34:13.121482+0000 mgr.y (mgr.25723) 882 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T06:34:14.518 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:14 smithi125 bash[80002]: cluster 2023-10-09T06:34:13.121482+0000 mgr.y (mgr.25723) 882 : cluster 0 pgmap v893: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T06:34:14.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:14.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:14.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:14.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:14.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:14.782 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 36585 36569 126.958 108.875 0.0194864 0.00787096 2023-10-09T06:34:14.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:14.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:14.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:14.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:15.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:34:15.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:15.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:15.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:34:15.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:15.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:15.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:15.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:15.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:15.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:15.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:15.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:15.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:15.621 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:15.622 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:15.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:15.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:15.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 37991 37975 124.9 87.875 0.00359109 0.00799947 2023-10-09T06:34:15.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:15.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:15.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:15.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:15.944 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:34:15.947 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:34:15.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:16.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:16.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:16.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:16.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:16.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:16.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:16 smithi155 bash[59495]: cluster 2023-10-09T06:34:15.122153+0000 mgr.y (mgr.25723) 883 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:34:16.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:16.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:16.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:16.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:16 smithi125 bash[77299]: cluster 2023-10-09T06:34:15.122153+0000 mgr.y (mgr.25723) 883 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:34:16.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:16 smithi125 bash[80002]: cluster 2023-10-09T06:34:15.122153+0000 mgr.y (mgr.25723) 883 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:16.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:16.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:16.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:16.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:16.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:34:16.777226+0000 min lat: 0.00131043 max lat: 0.352992 avg lat: 0.00810152 2023-10-09T06:34:16.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:34:16.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 39492 39476 123.345 93.8125 0.00606285 0.00810152 2023-10-09T06:34:16.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:16.817 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:16.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:16.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:16.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:17.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:17.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:17.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:17.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:17.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:17.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:17.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:17.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:17.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:17.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:17.584 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:17 smithi155 bash[59495]: audit 2023-10-09T06:34:15.922263+0000 mgr.y (mgr.25723) 884 : audit 0 from='client.45398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:17.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:17.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:17 smithi125 bash[77299]: audit 2023-10-09T06:34:15.922263+0000 mgr.y (mgr.25723) 884 : audit 0 from='client.45398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:17.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:17 smithi125 bash[80002]: audit 2023-10-09T06:34:15.922263+0000 mgr.y (mgr.25723) 884 : audit 0 from='client.45398 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:17.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:17.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:17.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 41100 41084 122.257 100.5 0.00714873 0.00817403 2023-10-09T06:34:17.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:17.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:17.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:17.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:17.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:18.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:18.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:18.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:18.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:18.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:18.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:18.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:18 smithi125 bash[77299]: cluster 2023-10-09T06:34:17.122690+0000 mgr.y (mgr.25723) 885 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:34:18.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:18 smithi125 bash[80002]: cluster 2023-10-09T06:34:17.122690+0000 mgr.y (mgr.25723) 885 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:34:18.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:18.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:18.512 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:18 smithi155 bash[59495]: cluster 2023-10-09T06:34:17.122690+0000 mgr.y (mgr.25723) 885 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:34:18.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:18.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:18.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:18.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:34:18.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:18.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 42627 42611 121.037 95.4375 0.0133926 0.00825588 2023-10-09T06:34:18.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:18.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:18.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:18.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:19.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:19.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:19.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:19.201 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":894,"stamp":"2023-10-09T06:34:15.121679+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2242385843,"num_objects":34171,"num_object_clones":0,"num_object_copies":102513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34171,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":35249,"num_write_kb":2200311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35313,"ondisk_log_size":35313,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9215968,"kb_used_data":6407776,"kb_used_omap":30,"kb_used_meta":2808097,"kb_avail":740745248,"statfs":{"total":767960285184,"available":758523133952,"internally_reserved":0,"allocated":6561562624,"data_stored":6556114325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":2875491439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1173225472,"num_objects":17902,"num_object_clones":0,"num_object_copies":53706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17902,"num_write_kb":1145728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002852"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":178,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195044+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:33:53.195044+0000","last_peered":"2023-10-09T06:33:53.195044+0000","last_clean":"2023-10-09T06:33:53.195044+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:33:53.195044+0000","last_undegraded":"2023-10-09T06:33:53.195044+0000","last_fullsized":"2023-10-09T06:33:53.195044+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179823+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:33:53.179823+0000","last_peered":"2023-10-09T06:33:53.179823+0000","last_clean":"2023-10-09T06:33:53.179823+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:33:53.179823+0000","last_undegraded":"2023-10-09T06:33:53.179823+0000","last_fullsized":"2023-10-09T06:33:53.179823+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188713+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:33:53.188713+0000","last_peered":"2023-10-09T06:33:53.188713+0000","last_clean":"2023-10-09T06:33:53.188713+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:33:53.188713+0000","last_undegraded":"2023-10-09T06:33:53.188713+0000","last_fullsized":"2023-10-09T06:33:53.188713+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190304+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:33:53.190304+0000","last_peered":"2023-10-09T06:33:53.190304+0000","last_clean":"2023-10-09T06:33:53.190304+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:33:53.190304+0000","last_undegraded":"2023-10-09T06:33:53.190304+0000","last_fullsized":"2023-10-09T06:33:53.190304+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:01.504835+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:34:01.504835+0000","last_peered":"2023-10-09T06:34:01.504835+0000","last_clean":"2023-10-09T06:34:01.504835+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:34:01.504835+0000","last_undegraded":"2023-10-09T06:34:01.504835+0000","last_fullsized":"2023-10-09T06:34:01.504835+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":37,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190244+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:33:53.190244+0000","last_peered":"2023-10-09T06:33:53.190244+0000","last_clean":"2023-10-09T06:33:53.190244+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:33:53.190244+0000","last_undegraded":"2023-10-09T06:33:53.190244+0000","last_fullsized":"2023-10-09T06:33:53.190244+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190334+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:33:53.190334+0000","last_peered":"2023-10-09T06:33:53.190334+0000","last_clean":"2023-10-09T06:33:53.190334+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:33:53.190334+0000","last_undegraded":"2023-10-09T06:33:53.190334+0000","last_fullsized":"2023-10-09T06:33:53.190334+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":204,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179435+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:33:53.179435+0000","last_peered":"2023-10-09T06:33:53.179435+0000","last_clean":"2023-10-09T06:33:53.179435+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:33:53.179435+0000","last_undegraded":"2023-10-09T06:33:53.179435+0000","last_fullsized":"2023-10-09T06:33:53.179435+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197591+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:33:53.197591+0000","last_peered":"2023-10-09T06:33:53.197591+0000","last_clean":"2023-10-09T06:33:53.197591+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:33:53.197591+0000","last_undegraded":"2023-10-09T06:33:53.197591+0000","last_fullsized":"2023-10-09T06:33:53.197591+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179205+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:33:53.179205+0000","last_peered":"2023-10-09T06:33:53.179205+0000","last_clean":"2023-10-09T06:33:53.179205+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:33:53.179205+0000","last_undegraded":"2023-10-09T06:33:53.179205+0000","last_fullsized":"2023-10-09T06:33:53.179205+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":95,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197512+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:33:53.197512+0000","last_peered":"2023-10-09T06:33:53.197512+0000","last_clean":"2023-10-09T06:33:53.197512+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:33:53.197512+0000","last_undegraded":"2023-10-09T06:33:53.197512+0000","last_fullsized":"2023-10-09T06:33:53.197512+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189308+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:33:53.189308+0000","last_peered":"2023-10-09T06:33:53.189308+0000","last_clean":"2023-10-09T06:33:53.189308+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:33:53.189308+0000","last_undegraded":"2023-10-09T06:33:53.189308+0000","last_fullsized":"2023-10-09T06:33:53.189308+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":245,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196341+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:33:53.196341+0000","last_peered":"2023-10-09T06:33:53.196341+0000","last_clean":"2023-10-09T06:33:53.196341+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:33:53.196341+0000","last_undegraded":"2023-10-09T06:33:53.196341+0000","last_fullsized":"2023-10-09T06:33:53.196341+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195973+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:33:53.195973+0000","last_peered":"2023-10-09T06:33:53.195973+0000","last_clean":"2023-10-09T06:33:53.195973+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:33:53.195973+0000","last_undegraded":"2023-10-09T06:33:53.195973+0000","last_fullsized":"2023-10-09T06:33:53.195973+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196675+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:33:53.196675+0000","last_peered":"2023-10-09T06:33:53.196675+0000","last_clean":"2023-10-09T06:33:53.196675+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:33:53.196675+0000","last_undegraded":"2023-10-09T06:33:53.196675+0000","last_fullsized":"2023-10-09T06:33:53.196675+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":191,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179490+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:33:53.179490+0000","last_peered":"2023-10-09T06:33:53.179490+0000","last_clean":"2023-10-09T06:33:53.179490+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:33:53.179490+0000","last_undegraded":"2023-10-09T06:33:53.179490+0000","last_fullsized":"2023-10-09T06:33:53.179490+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":171,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215283+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:34:08.215283+0000","last_peered":"2023-10-09T06:34:08.215283+0000","last_clean":"2023-10-09T06:34:08.215283+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:34:08.215283+0000","last_undegraded":"2023-10-09T06:34:08.215283+0000","last_fullsized":"2023-10-09T06:34:08.215283+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194987+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:33:53.194987+0000","last_peered":"2023-10-09T06:33:53.194987+0000","last_clean":"2023-10-09T06:33:53.194987+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:33:53.194987+0000","last_undegraded":"2023-10-09T06:33:53.194987+0000","last_fullsized":"2023-10-09T06:33:53.194987+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196318+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:33:53.196318+0000","last_peered":"2023-10-09T06:33:53.196318+0000","last_clean":"2023-10-09T06:33:53.196318+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:33:53.196318+0000","last_undegraded":"2023-10-09T06:33:53.196318+0000","last_fullsized":"2023-10-09T06:33:53.196318+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190261+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:33:53.190261+0000","last_peered":"2023-10-09T06:33:53.190261+0000","last_clean":"2023-10-09T06:33:53.190261+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:33:53.190261+0000","last_undegraded":"2023-10-09T06:33:53.190261+0000","last_fullsized":"2023-10-09T06:33:53.190261+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":47,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189348+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:33:53.189348+0000","last_peered":"2023-10-09T06:33:53.189348+0000","last_clean":"2023-10-09T06:33:53.189348+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:33:53.189348+0000","last_undegraded":"2023-10-09T06:33:53.189348+0000","last_fullsized":"2023-10-09T06:33:53.189348+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":204,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:01.504915+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:34:01.504915+0000","last_peered":"2023-10-09T06:34:01.504915+0000","last_clean":"2023-10-09T06:34:01.504915+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:34:01.504915+0000","last_undegraded":"2023-10-09T06:34:01.504915+0000","last_fullsized":"2023-10-09T06:34:01.504915+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"197'2256","reported_seq":2269,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.662834+0000","last_change":"2023-10-09T06:33:33.324346+0000","last_active":"2023-10-09T06:34:14.662834+0000","last_peered":"2023-10-09T06:34:14.662834+0000","last_clean":"2023-10-09T06:34:14.662834+0000","last_became_active":"2023-10-09T06:33:33.323046+0000","last_became_peered":"2023-10-09T06:33:33.323046+0000","last_unstale":"2023-10-09T06:34:14.662834+0000","last_undegraded":"2023-10-09T06:34:14.662834+0000","last_fullsized":"2023-10-09T06:34:14.662834+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2256,"log_dups_size":0,"ondisk_log_size":2256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147849216,"num_objects":2256,"num_object_clones":0,"num_object_copies":6768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2256,"num_write_kb":144384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":93,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195388+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:33:53.195388+0000","last_peered":"2023-10-09T06:33:53.195388+0000","last_clean":"2023-10-09T06:33:53.195388+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:33:53.195388+0000","last_undegraded":"2023-10-09T06:33:53.195388+0000","last_fullsized":"2023-10-09T06:33:53.195388+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":182,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195691+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:33:53.195691+0000","last_peered":"2023-10-09T06:33:53.195691+0000","last_clean":"2023-10-09T06:33:53.195691+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:33:53.195691+0000","last_undegraded":"2023-10-09T06:33:53.195691+0000","last_fullsized":"2023-10-09T06:33:53.195691+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"197'2270","reported_seq":2283,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.936925+0000","last_change":"2023-10-09T06:33:33.593835+0000","last_active":"2023-10-09T06:34:14.936925+0000","last_peered":"2023-10-09T06:34:14.936925+0000","last_clean":"2023-10-09T06:34:14.936925+0000","last_became_active":"2023-10-09T06:33:33.593363+0000","last_became_peered":"2023-10-09T06:33:33.593363+0000","last_unstale":"2023-10-09T06:34:14.936925+0000","last_undegraded":"2023-10-09T06:34:14.936925+0000","last_fullsized":"2023-10-09T06:34:14.936925+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2270,"log_dups_size":0,"ondisk_log_size":2270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148766720,"num_objects":2270,"num_object_clones":0,"num_object_copies":6810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2270,"num_write_kb":145280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":93,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197341+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:33:53.197341+0000","last_peered":"2023-10-09T06:33:53.197341+0000","last_clean":"2023-10-09T06:33:53.197341+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:33:53.197341+0000","last_undegraded":"2023-10-09T06:33:53.197341+0000","last_fullsized":"2023-10-09T06:33:53.197341+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194805+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:33:53.194805+0000","last_peered":"2023-10-09T06:33:53.194805+0000","last_clean":"2023-10-09T06:33:53.194805+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:33:53.194805+0000","last_undegraded":"2023-10-09T06:33:53.194805+0000","last_fullsized":"2023-10-09T06:33:53.194805+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"197'2144","reported_seq":2158,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.689362+0000","last_change":"2023-10-09T06:33:33.587458+0000","last_active":"2023-10-09T06:34:13.689362+0000","last_peered":"2023-10-09T06:34:13.689362+0000","last_clean":"2023-10-09T06:34:13.689362+0000","last_became_active":"2023-10-09T06:33:33.586581+0000","last_became_peered":"2023-10-09T06:33:33.586581+0000","last_unstale":"2023-10-09T06:34:13.689362+0000","last_undegraded":"2023-10-09T06:34:13.689362+0000","last_fullsized":"2023-10-09T06:34:13.689362+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2144,"log_dups_size":0,"ondisk_log_size":2144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.486216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2144,"num_write_kb":137216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":119,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197152+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:33:53.197152+0000","last_peered":"2023-10-09T06:33:53.197152+0000","last_clean":"2023-10-09T06:33:53.197152+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:33:53.197152+0000","last_undegraded":"2023-10-09T06:33:53.197152+0000","last_fullsized":"2023-10-09T06:33:53.197152+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":37,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189419+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:33:53.189419+0000","last_peered":"2023-10-09T06:33:53.189419+0000","last_clean":"2023-10-09T06:33:53.189419+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:33:53.189419+0000","last_undegraded":"2023-10-09T06:33:53.189419+0000","last_fullsized":"2023-10-09T06:33:53.189419+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"197'2078","reported_seq":2092,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.425905+0000","last_change":"2023-10-09T06:33:33.319454+0000","last_active":"2023-10-09T06:34:12.425905+0000","last_peered":"2023-10-09T06:34:12.425905+0000","last_clean":"2023-10-09T06:34:12.425905+0000","last_became_active":"2023-10-09T06:33:33.318032+0000","last_became_peered":"2023-10-09T06:33:33.318032+0000","last_unstale":"2023-10-09T06:34:12.425905+0000","last_undegraded":"2023-10-09T06:34:12.425905+0000","last_fullsized":"2023-10-09T06:34:12.425905+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2078,"log_dups_size":0,"ondisk_log_size":2078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:46.906949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136183808,"num_objects":2078,"num_object_clones":0,"num_object_copies":6234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2078,"num_write_kb":132992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195345+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:33:53.195345+0000","last_peered":"2023-10-09T06:33:53.195345+0000","last_clean":"2023-10-09T06:33:53.195345+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:33:53.195345+0000","last_undegraded":"2023-10-09T06:33:53.195345+0000","last_fullsized":"2023-10-09T06:33:53.195345+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189796+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:33:53.189796+0000","last_peered":"2023-10-09T06:33:53.189796+0000","last_clean":"2023-10-09T06:33:53.189796+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:33:53.189796+0000","last_undegraded":"2023-10-09T06:33:53.189796+0000","last_fullsized":"2023-10-09T06:33:53.189796+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"197'1930","reported_seq":1944,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.501534+0000","last_change":"2023-10-09T06:33:33.320528+0000","last_active":"2023-10-09T06:34:11.501534+0000","last_peered":"2023-10-09T06:34:11.501534+0000","last_clean":"2023-10-09T06:34:11.501534+0000","last_became_active":"2023-10-09T06:33:33.320042+0000","last_became_peered":"2023-10-09T06:33:33.320042+0000","last_unstale":"2023-10-09T06:34:11.501534+0000","last_undegraded":"2023-10-09T06:34:11.501534+0000","last_fullsized":"2023-10-09T06:34:11.501534+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":1930,"log_dups_size":0,"ondisk_log_size":1930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.099867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126484480,"num_objects":1930,"num_object_clones":0,"num_object_copies":5790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1930,"num_write_kb":123520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179796+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:33:53.179796+0000","last_peered":"2023-10-09T06:33:53.179796+0000","last_clean":"2023-10-09T06:33:53.179796+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:33:53.179796+0000","last_undegraded":"2023-10-09T06:33:53.179796+0000","last_fullsized":"2023-10-09T06:33:53.179796+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196072+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:33:53.196072+0000","last_peered":"2023-10-09T06:33:53.196072+0000","last_clean":"2023-10-09T06:33:53.196072+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:33:53.196072+0000","last_undegraded":"2023-10-09T06:33:53.196072+0000","last_fullsized":"2023-10-09T06:33:53.196072+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"197'2041","reported_seq":2055,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.407501+0000","last_change":"2023-10-09T06:33:33.321209+0000","last_active":"2023-10-09T06:34:12.407501+0000","last_peered":"2023-10-09T06:34:12.407501+0000","last_clean":"2023-10-09T06:34:12.407501+0000","last_became_active":"2023-10-09T06:33:33.320866+0000","last_became_peered":"2023-10-09T06:33:33.320866+0000","last_unstale":"2023-10-09T06:34:12.407501+0000","last_undegraded":"2023-10-09T06:34:12.407501+0000","last_fullsized":"2023-10-09T06:34:12.407501+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2041,"log_dups_size":0,"ondisk_log_size":2041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:34.914529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133758976,"num_objects":2041,"num_object_clones":0,"num_object_copies":6123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2041,"num_write_kb":130624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195254+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:33:53.195254+0000","last_peered":"2023-10-09T06:33:53.195254+0000","last_clean":"2023-10-09T06:33:53.195254+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:33:53.195254+0000","last_undegraded":"2023-10-09T06:33:53.195254+0000","last_fullsized":"2023-10-09T06:33:53.195254+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196465+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:33:53.196465+0000","last_peered":"2023-10-09T06:33:53.196465+0000","last_clean":"2023-10-09T06:33:53.196465+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:33:53.196465+0000","last_undegraded":"2023-10-09T06:33:53.196465+0000","last_fullsized":"2023-10-09T06:33:53.196465+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"197'2311","reported_seq":2324,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.712343+0000","last_change":"2023-10-09T06:33:33.244614+0000","last_active":"2023-10-09T06:34:14.712343+0000","last_peered":"2023-10-09T06:34:14.712343+0000","last_clean":"2023-10-09T06:34:14.712343+0000","last_became_active":"2023-10-09T06:33:33.243719+0000","last_became_peered":"2023-10-09T06:33:33.243719+0000","last_unstale":"2023-10-09T06:34:14.712343+0000","last_undegraded":"2023-10-09T06:34:14.712343+0000","last_fullsized":"2023-10-09T06:34:14.712343+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2311,"log_dups_size":0,"ondisk_log_size":2311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2311,"num_write_kb":147904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194482+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:33:53.194482+0000","last_peered":"2023-10-09T06:33:53.194482+0000","last_clean":"2023-10-09T06:33:53.194482+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:33:53.194482+0000","last_undegraded":"2023-10-09T06:33:53.194482+0000","last_fullsized":"2023-10-09T06:33:53.194482+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196345+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:33:53.196345+0000","last_peered":"2023-10-09T06:33:53.196345+0000","last_clean":"2023-10-09T06:33:53.196345+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:33:53.196345+0000","last_undegraded":"2023-10-09T06:33:53.196345+0000","last_fullsized":"2023-10-09T06:33:53.196345+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"197'1992","reported_seq":2004,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.429585+0000","last_change":"2023-10-09T06:33:33.319285+0000","last_active":"2023-10-09T06:34:12.429585+0000","last_peered":"2023-10-09T06:34:12.429585+0000","last_clean":"2023-10-09T06:34:12.429585+0000","last_became_active":"2023-10-09T06:33:33.318947+0000","last_became_peered":"2023-10-09T06:33:33.318947+0000","last_unstale":"2023-10-09T06:34:12.429585+0000","last_undegraded":"2023-10-09T06:34:12.429585+0000","last_fullsized":"2023-10-09T06:34:12.429585+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":1992,"log_dups_size":0,"ondisk_log_size":1992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:22.467861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1992,"num_write_kb":127488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":193,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.219256+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:34:08.219256+0000","last_peered":"2023-10-09T06:34:08.219256+0000","last_clean":"2023-10-09T06:34:08.219256+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:34:08.219256+0000","last_undegraded":"2023-10-09T06:34:08.219256+0000","last_fullsized":"2023-10-09T06:34:08.219256+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:57.404378+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:33:57.404378+0000","last_peered":"2023-10-09T06:33:57.404378+0000","last_clean":"2023-10-09T06:33:57.404378+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:33:57.404378+0000","last_undegraded":"2023-10-09T06:33:57.404378+0000","last_fullsized":"2023-10-09T06:33:57.404378+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"197'2205","reported_seq":2219,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.698034+0000","last_change":"2023-10-09T06:33:33.317867+0000","last_active":"2023-10-09T06:34:13.698034+0000","last_peered":"2023-10-09T06:34:13.698034+0000","last_clean":"2023-10-09T06:34:13.698034+0000","last_became_active":"2023-10-09T06:33:33.317504+0000","last_became_peered":"2023-10-09T06:33:33.317504+0000","last_unstale":"2023-10-09T06:34:13.698034+0000","last_undegraded":"2023-10-09T06:34:13.698034+0000","last_fullsized":"2023-10-09T06:34:13.698034+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2205,"log_dups_size":0,"ondisk_log_size":2205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:56.906981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144506880,"num_objects":2205,"num_object_clones":0,"num_object_copies":6615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2205,"num_write_kb":141120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1050,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189058+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:33:53.189058+0000","last_peered":"2023-10-09T06:33:53.189058+0000","last_clean":"2023-10-09T06:33:53.189058+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:33:53.189058+0000","last_undegraded":"2023-10-09T06:33:53.189058+0000","last_fullsized":"2023-10-09T06:33:53.189058+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197546+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:33:53.197546+0000","last_peered":"2023-10-09T06:33:53.197546+0000","last_clean":"2023-10-09T06:33:53.197546+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:33:53.197546+0000","last_undegraded":"2023-10-09T06:33:53.197546+0000","last_fullsized":"2023-10-09T06:33:53.197546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189192+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:33:53.189192+0000","last_peered":"2023-10-09T06:33:53.189192+0000","last_clean":"2023-10-09T06:33:53.189192+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:33:53.189192+0000","last_undegraded":"2023-10-09T06:33:53.189192+0000","last_fullsized":"2023-10-09T06:33:53.189192+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"197'2264","reported_seq":2277,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.652634+0000","last_change":"2023-10-09T06:33:33.593915+0000","last_active":"2023-10-09T06:34:14.652634+0000","last_peered":"2023-10-09T06:34:14.652634+0000","last_clean":"2023-10-09T06:34:14.652634+0000","last_became_active":"2023-10-09T06:33:33.593144+0000","last_became_peered":"2023-10-09T06:33:33.593144+0000","last_unstale":"2023-10-09T06:34:14.652634+0000","last_undegraded":"2023-10-09T06:34:14.652634+0000","last_fullsized":"2023-10-09T06:34:14.652634+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2264,"log_dups_size":0,"ondisk_log_size":2264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2264,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":835,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195371+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:33:53.195371+0000","last_peered":"2023-10-09T06:33:53.195371+0000","last_clean":"2023-10-09T06:33:53.195371+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:33:53.195371+0000","last_undegraded":"2023-10-09T06:33:53.195371+0000","last_fullsized":"2023-10-09T06:33:53.195371+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195951+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:33:53.195951+0000","last_peered":"2023-10-09T06:33:53.195951+0000","last_clean":"2023-10-09T06:33:53.195951+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:33:53.195951+0000","last_undegraded":"2023-10-09T06:33:53.195951+0000","last_fullsized":"2023-10-09T06:33:53.195951+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188874+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:33:53.188874+0000","last_peered":"2023-10-09T06:33:53.188874+0000","last_clean":"2023-10-09T06:33:53.188874+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:33:53.188874+0000","last_undegraded":"2023-10-09T06:33:53.188874+0000","last_fullsized":"2023-10-09T06:33:53.188874+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"197'1846","reported_seq":1860,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.514851+0000","last_change":"2023-10-09T06:33:33.319622+0000","last_active":"2023-10-09T06:34:11.514851+0000","last_peered":"2023-10-09T06:34:11.514851+0000","last_clean":"2023-10-09T06:34:11.514851+0000","last_became_active":"2023-10-09T06:33:33.319140+0000","last_became_peered":"2023-10-09T06:33:33.319140+0000","last_unstale":"2023-10-09T06:34:11.514851+0000","last_undegraded":"2023-10-09T06:34:11.514851+0000","last_fullsized":"2023-10-09T06:34:11.514851+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":1846,"log_dups_size":0,"ondisk_log_size":1846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:00.867299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120979456,"num_objects":1846,"num_object_clones":0,"num_object_copies":5538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1846,"num_write_kb":118144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":561,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189207+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:33:53.189207+0000","last_peered":"2023-10-09T06:33:53.189207+0000","last_clean":"2023-10-09T06:33:53.189207+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:33:53.189207+0000","last_undegraded":"2023-10-09T06:33:53.189207+0000","last_fullsized":"2023-10-09T06:33:53.189207+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188520+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:33:53.188520+0000","last_peered":"2023-10-09T06:33:53.188520+0000","last_clean":"2023-10-09T06:33:53.188520+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:33:53.188520+0000","last_undegraded":"2023-10-09T06:33:53.188520+0000","last_fullsized":"2023-10-09T06:33:53.188520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190699+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:33:53.190699+0000","last_peered":"2023-10-09T06:33:53.190699+0000","last_clean":"2023-10-09T06:33:53.190699+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:33:53.190699+0000","last_undegraded":"2023-10-09T06:33:53.190699+0000","last_fullsized":"2023-10-09T06:33:53.190699+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"197'2316","reported_seq":2327,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.674283+0000","last_change":"2023-10-09T06:33:33.244731+0000","last_active":"2023-10-09T06:34:14.674283+0000","last_peered":"2023-10-09T06:34:14.674283+0000","last_clean":"2023-10-09T06:34:14.674283+0000","last_became_active":"2023-10-09T06:33:33.243726+0000","last_became_peered":"2023-10-09T06:33:33.243726+0000","last_unstale":"2023-10-09T06:34:14.674283+0000","last_undegraded":"2023-10-09T06:34:14.674283+0000","last_fullsized":"2023-10-09T06:34:14.674283+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2316,"log_dups_size":0,"ondisk_log_size":2316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151781376,"num_objects":2316,"num_object_clones":0,"num_object_copies":6948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2316,"num_write_kb":148224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":545,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194774+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:33:53.194774+0000","last_peered":"2023-10-09T06:33:53.194774+0000","last_clean":"2023-10-09T06:33:53.194774+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:33:53.194774+0000","last_undegraded":"2023-10-09T06:33:53.194774+0000","last_fullsized":"2023-10-09T06:33:53.194774+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":439,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195883+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:33:53.195883+0000","last_peered":"2023-10-09T06:33:53.195883+0000","last_clean":"2023-10-09T06:33:53.195883+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:33:53.195883+0000","last_undegraded":"2023-10-09T06:33:53.195883+0000","last_fullsized":"2023-10-09T06:33:53.195883+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":183,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179196+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:33:53.179196+0000","last_peered":"2023-10-09T06:33:53.179196+0000","last_clean":"2023-10-09T06:33:53.179196+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:33:53.179196+0000","last_undegraded":"2023-10-09T06:33:53.179196+0000","last_fullsized":"2023-10-09T06:33:53.179196+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196594+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:33:53.196594+0000","last_peered":"2023-10-09T06:33:53.196594+0000","last_clean":"2023-10-09T06:33:53.196594+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:33:53.196594+0000","last_undegraded":"2023-10-09T06:33:53.196594+0000","last_fullsized":"2023-10-09T06:33:53.196594+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"197'2062","reported_seq":2076,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.402566+0000","last_change":"2023-10-09T06:33:33.319537+0000","last_active":"2023-10-09T06:34:12.402566+0000","last_peered":"2023-10-09T06:34:12.402566+0000","last_clean":"2023-10-09T06:34:12.402566+0000","last_became_active":"2023-10-09T06:33:33.319150+0000","last_became_peered":"2023-10-09T06:33:33.319150+0000","last_unstale":"2023-10-09T06:34:12.402566+0000","last_undegraded":"2023-10-09T06:34:12.402566+0000","last_fullsized":"2023-10-09T06:34:12.402566+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2062,"log_dups_size":0,"ondisk_log_size":2062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:18.214528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135135232,"num_objects":2062,"num_object_clones":0,"num_object_copies":6186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2062,"num_write_kb":131968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":118,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195494+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:33:53.195494+0000","last_peered":"2023-10-09T06:33:53.195494+0000","last_clean":"2023-10-09T06:33:53.195494+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:33:53.195494+0000","last_undegraded":"2023-10-09T06:33:53.195494+0000","last_fullsized":"2023-10-09T06:33:53.195494+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713035+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:34:03.713035+0000","last_peered":"2023-10-09T06:34:03.713035+0000","last_clean":"2023-10-09T06:34:03.713035+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:34:03.713035+0000","last_undegraded":"2023-10-09T06:34:03.713035+0000","last_fullsized":"2023-10-09T06:34:03.713035+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190411+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:33:53.190411+0000","last_peered":"2023-10-09T06:33:53.190411+0000","last_clean":"2023-10-09T06:33:53.190411+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:33:53.190411+0000","last_undegraded":"2023-10-09T06:33:53.190411+0000","last_fullsized":"2023-10-09T06:33:53.190411+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"197'1908","reported_seq":1922,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.515922+0000","last_change":"2023-10-09T06:33:33.320380+0000","last_active":"2023-10-09T06:34:11.515922+0000","last_peered":"2023-10-09T06:34:11.515922+0000","last_clean":"2023-10-09T06:34:11.515922+0000","last_became_active":"2023-10-09T06:33:33.319960+0000","last_became_peered":"2023-10-09T06:33:33.319960+0000","last_unstale":"2023-10-09T06:34:11.515922+0000","last_undegraded":"2023-10-09T06:34:11.515922+0000","last_fullsized":"2023-10-09T06:34:11.515922+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":1908,"log_dups_size":0,"ondisk_log_size":1908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:25.704605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1908,"num_write_kb":122112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":107,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197259+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:33:53.197259+0000","last_peered":"2023-10-09T06:33:53.197259+0000","last_clean":"2023-10-09T06:33:53.197259+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:33:53.197259+0000","last_undegraded":"2023-10-09T06:33:53.197259+0000","last_fullsized":"2023-10-09T06:33:53.197259+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":149,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188965+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:33:53.188965+0000","last_peered":"2023-10-09T06:33:53.188965+0000","last_clean":"2023-10-09T06:33:53.188965+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:33:53.188965+0000","last_undegraded":"2023-10-09T06:33:53.188965+0000","last_fullsized":"2023-10-09T06:33:53.188965+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":166,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713069+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:34:03.713069+0000","last_peered":"2023-10-09T06:34:03.713069+0000","last_clean":"2023-10-09T06:34:03.713069+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:34:03.713069+0000","last_undegraded":"2023-10-09T06:34:03.713069+0000","last_fullsized":"2023-10-09T06:34:03.713069+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"197'2282","reported_seq":2296,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.695897+0000","last_change":"2023-10-09T06:33:33.586554+0000","last_active":"2023-10-09T06:34:13.695897+0000","last_peered":"2023-10-09T06:34:13.695897+0000","last_clean":"2023-10-09T06:34:13.695897+0000","last_became_active":"2023-10-09T06:33:33.586066+0000","last_became_peered":"2023-10-09T06:33:33.586066+0000","last_unstale":"2023-10-09T06:34:13.695897+0000","last_undegraded":"2023-10-09T06:34:13.695897+0000","last_fullsized":"2023-10-09T06:34:13.695897+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2282,"log_dups_size":0,"ondisk_log_size":2282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.973752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149553152,"num_objects":2282,"num_object_clones":0,"num_object_copies":6846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2282,"num_write_kb":146048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":199,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.228427+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:34:08.228427+0000","last_peered":"2023-10-09T06:34:08.228427+0000","last_clean":"2023-10-09T06:34:08.228427+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:34:08.228427+0000","last_undegraded":"2023-10-09T06:34:08.228427+0000","last_fullsized":"2023-10-09T06:34:08.228427+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":580,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190686+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:33:53.190686+0000","last_peered":"2023-10-09T06:33:53.190686+0000","last_clean":"2023-10-09T06:33:53.190686+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:33:53.190686+0000","last_undegraded":"2023-10-09T06:33:53.190686+0000","last_fullsized":"2023-10-09T06:33:53.190686+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179154+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:33:53.179154+0000","last_peered":"2023-10-09T06:33:53.179154+0000","last_clean":"2023-10-09T06:33:53.179154+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:33:53.179154+0000","last_undegraded":"2023-10-09T06:33:53.179154+0000","last_fullsized":"2023-10-09T06:33:53.179154+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"197'2238","reported_seq":2251,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:14.661511+0000","last_change":"2023-10-09T06:33:33.594495+0000","last_active":"2023-10-09T06:34:14.661511+0000","last_peered":"2023-10-09T06:34:14.661511+0000","last_clean":"2023-10-09T06:34:14.661511+0000","last_became_active":"2023-10-09T06:33:33.593206+0000","last_became_peered":"2023-10-09T06:33:33.593206+0000","last_unstale":"2023-10-09T06:34:14.661511+0000","last_undegraded":"2023-10-09T06:34:14.661511+0000","last_fullsized":"2023-10-09T06:34:14.661511+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":2238,"log_dups_size":0,"ondisk_log_size":2238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146669568,"num_objects":2238,"num_object_clones":0,"num_object_copies":6714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2238,"num_write_kb":143232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":717,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190257+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:33:53.190257+0000","last_peered":"2023-10-09T06:33:53.190257+0000","last_clean":"2023-10-09T06:33:53.190257+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:33:53.190257+0000","last_undegraded":"2023-10-09T06:33:53.190257+0000","last_fullsized":"2023-10-09T06:33:53.190257+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190347+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:33:53.190347+0000","last_peered":"2023-10-09T06:33:53.190347+0000","last_clean":"2023-10-09T06:33:53.190347+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:33:53.190347+0000","last_undegraded":"2023-10-09T06:33:53.190347+0000","last_fullsized":"2023-10-09T06:33:53.190347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.199037+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:33:53.199037+0000","last_peered":"2023-10-09T06:33:53.199037+0000","last_clean":"2023-10-09T06:33:53.199037+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:33:53.199037+0000","last_undegraded":"2023-10-09T06:33:53.199037+0000","last_fullsized":"2023-10-09T06:33:53.199037+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189956+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:33:53.189956+0000","last_peered":"2023-10-09T06:33:53.189956+0000","last_clean":"2023-10-09T06:33:53.189956+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:33:53.189956+0000","last_undegraded":"2023-10-09T06:33:53.189956+0000","last_fullsized":"2023-10-09T06:33:53.189956+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.222382+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:34:08.222382+0000","last_peered":"2023-10-09T06:34:08.222382+0000","last_clean":"2023-10-09T06:34:08.222382+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:34:08.222382+0000","last_undegraded":"2023-10-09T06:34:08.222382+0000","last_fullsized":"2023-10-09T06:34:08.222382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":164,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195587+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:33:53.195587+0000","last_peered":"2023-10-09T06:33:53.195587+0000","last_clean":"2023-10-09T06:33:53.195587+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:33:53.195587+0000","last_undegraded":"2023-10-09T06:33:53.195587+0000","last_fullsized":"2023-10-09T06:33:53.195587+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196153+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:33:53.196153+0000","last_peered":"2023-10-09T06:33:53.196153+0000","last_clean":"2023-10-09T06:33:53.196153+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:33:53.196153+0000","last_undegraded":"2023-10-09T06:33:53.196153+0000","last_fullsized":"2023-10-09T06:33:53.196153+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":175,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197544+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:33:53.197544+0000","last_peered":"2023-10-09T06:33:53.197544+0000","last_clean":"2023-10-09T06:33:53.197544+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:33:53.197544+0000","last_undegraded":"2023-10-09T06:33:53.197544+0000","last_fullsized":"2023-10-09T06:33:53.197544+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195444+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:33:53.195444+0000","last_peered":"2023-10-09T06:33:53.195444+0000","last_clean":"2023-10-09T06:33:53.195444+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:33:53.195444+0000","last_undegraded":"2023-10-09T06:33:53.195444+0000","last_fullsized":"2023-10-09T06:33:53.195444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":198,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179885+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:33:53.179885+0000","last_peered":"2023-10-09T06:33:53.179885+0000","last_clean":"2023-10-09T06:33:53.179885+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:33:53.179885+0000","last_undegraded":"2023-10-09T06:33:53.179885+0000","last_fullsized":"2023-10-09T06:33:53.179885+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:06.507183+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:34:06.507183+0000","last_peered":"2023-10-09T06:34:06.507183+0000","last_clean":"2023-10-09T06:34:06.507183+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:34:06.507183+0000","last_undegraded":"2023-10-09T06:34:06.507183+0000","last_fullsized":"2023-10-09T06:34:06.507183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":158,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713130+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:34:03.713130+0000","last_peered":"2023-10-09T06:34:03.713130+0000","last_clean":"2023-10-09T06:34:03.713130+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:34:03.713130+0000","last_undegraded":"2023-10-09T06:34:03.713130+0000","last_fullsized":"2023-10-09T06:34:03.713130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194395+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:33:53.194395+0000","last_peered":"2023-10-09T06:33:53.194395+0000","last_clean":"2023-10-09T06:33:53.194395+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:33:53.194395+0000","last_undegraded":"2023-10-09T06:33:53.194395+0000","last_fullsized":"2023-10-09T06:33:53.194395+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2237595648,"num_objects":34143,"num_object_clones":0,"num_object_copies":102429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34143,"num_write_kb":2185152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6750863360,"data_stored":6750863360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":34143,"ondisk_log_size":34143,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430635,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1019492,"kb_used_data":709980,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92725660,"statfs":{"total":95995035648,"available":94951075840,"internally_reserved":0,"allocated":727019520,"data_stored":726462397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":3.8769999999999998,"15min":2.9300000000000002},"min":{"1min":0.50700000000000001,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":1.367,"5min":178.738,"15min":178.738},"last":0.371},{"interface":"front","average":{"1min":0.78100000000000003,"5min":3.8719999999999999,"15min":2.8690000000000002},"min":{"1min":0.55000000000000004,"5min":0.252,"15min":0.252},"max":{"1min":1.1950000000000001,"5min":177.75,"15min":177.75},"last":0.29999999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.2070000000000001,"15min":2.0329999999999999},"min":{"1min":0.35699999999999998,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":4.2190000000000003,"5min":7.4740000000000002,"15min":104.825},"last":2.6829999999999998},{"interface":"front","average":{"1min":1.101,"5min":1.3819999999999999,"15min":1.9350000000000001},"min":{"1min":0.67900000000000005,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.5030000000000001,"5min":7.9119999999999999,"15min":104.637},"last":2.7120000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.3999999999999999,"15min":2.4300000000000002},"min":{"1min":0.53300000000000003,"5min":0.36199999999999999,"15min":0.29399999999999998},"max":{"1min":1.1859999999999999,"5min":15.585000000000001,"15min":104.992},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.371,"15min":2.327},"min":{"1min":0.69799999999999995,"5min":0.307,"15min":0.307},"max":{"1min":1.3859999999999999,"5min":7.0629999999999997,"15min":104.691},"last":0.90700000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":5.2640000000000002,"15min":3.552},"min":{"1min":0.44,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":1.28,"5min":183.971,"15min":183.971},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":5.0650000000000004,"15min":3.351},"min":{"1min":0.40000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.9610000000000001,"5min":183.095,"15min":183.095},"last":0.60399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.018,"15min":1.2370000000000001},"min":{"1min":0.57699999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.397,"5min":6.6239999999999997,"15min":11.029999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.002,"5min":1.298,"15min":1.5980000000000001},"min":{"1min":0.54000000000000004,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.341,"5min":15.504,"15min":15.504},"last":0.36099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0449999999999999,"15min":1.3999999999999999},"min":{"1min":0.626,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.4450000000000001,"5min":7.1550000000000002,"15min":11.996},"last":0.876},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.012,"15min":1.3939999999999999},"min":{"1min":0.64200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.631,"5min":5.4909999999999997,"15min":13.634},"last":4.7309999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.4259999999999999,"15min":1.3260000000000001},"min":{"1min":0.64900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":1.6879999999999999,"5min":15.521000000000001,"15min":15.521000000000001},"last":2.1349999999999998},{"interface":"front","average":{"1min":1.115,"5min":1.347,"15min":1.5049999999999999},"min":{"1min":0.496,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":1.5109999999999999,"5min":15.449999999999999,"15min":15.449999999999999},"last":1.21}]}]},{"osd":6,"up_from":42,"seq":180388626867,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036308,"kb_used_data":730892,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92708844,"statfs":{"total":95995035648,"available":94933856256,"internally_reserved":0,"allocated":748433408,"data_stored":747891912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.319,"15min":1.379},"min":{"1min":0.64800000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":1.2969999999999999,"5min":9.4649999999999999,"15min":32.289999999999999},"last":3.9119999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.2649999999999999,"15min":1.48},"min":{"1min":0.31900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.2270000000000001,"5min":6.4240000000000004,"15min":31.454999999999998},"last":0.504}]},{"osd":1,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.4339999999999999,"15min":1.819},"min":{"1min":0.42499999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":5.3760000000000003,"5min":6.9710000000000001,"15min":137.93899999999999},"last":4.1210000000000004},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.4410000000000001,"15min":1.845},"min":{"1min":0.54400000000000004,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":4.0650000000000004,"5min":11.281000000000001,"15min":138.02500000000001},"last":4.032}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":2.4740000000000002,"15min":2.3900000000000001},"min":{"1min":0.56000000000000005,"5min":0.35699999999999998,"15min":0.32000000000000001},"max":{"1min":2.2839999999999998,"5min":53.616,"15min":95.540000000000006},"last":3.9580000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":2.6120000000000001,"15min":2.3650000000000002},"min":{"1min":0.61099999999999999,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":1.454,"5min":53.665999999999997,"15min":96.352000000000004},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.23,"15min":2.3599999999999999},"min":{"1min":0.77300000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":2.468,"5min":27.491,"15min":115.09399999999999},"last":3.9980000000000002},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.8380000000000001,"15min":2.0089999999999999},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":1.3009999999999999,"5min":26.815000000000001,"15min":112.886},"last":4.2450000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":0.95799999999999996,"15min":1.2749999999999999},"min":{"1min":0.435,"5min":0.314,"15min":0.26000000000000001},"max":{"1min":1.478,"5min":6.3799999999999999,"15min":7.3090000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.071,"15min":1.4379999999999999},"min":{"1min":0.53500000000000003,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":4.1600000000000001,"5min":7.0060000000000002,"15min":10.300000000000001},"last":4.0890000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.91600000000000004,"15min":1.4510000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":1.51,"5min":6.4850000000000003,"15min":11.754},"last":0.441},{"interface":"front","average":{"1min":1.274,"5min":1.119,"15min":1.571},"min":{"1min":0.45200000000000001,"5min":0.38200000000000001,"15min":0.308},"max":{"1min":4.1950000000000003,"5min":12.215999999999999,"15min":12.295999999999999},"last":4.0549999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.083,"15min":1.502},"min":{"1min":0.81299999999999994,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":1.651,"5min":6.5110000000000001,"15min":11.076000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.27,"5min":1.127,"15min":1.4330000000000001},"min":{"1min":0.94999999999999996,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":2.52,"5min":9.4179999999999993,"15min":11.102},"last":3.859}]}]},{"osd":1,"up_from":93,"seq":399431958776,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209364,"kb_used_data":864652,"kb_used_omap":7,"kb_used_meta":344696,"kb_avail":92535788,"statfs":{"total":95995035648,"available":94756646912,"internally_reserved":0,"allocated":885403648,"data_stored":884571608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":352969104},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":8.2799999999999994,"5min":4.843,"15min":2.556},"min":{"1min":0.217,"5min":0.187,"15min":0.187},"max":{"1min":132.886,"5min":132.886,"15min":132.886},"last":2.2730000000000001},{"interface":"front","average":{"1min":8.5730000000000004,"5min":4.8529999999999998,"15min":2.7639999999999998},"min":{"1min":0.40500000000000003,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":133.315,"5min":133.315,"15min":133.315},"last":2.556}]},{"osd":2,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.6440000000000001,"15min":1.8740000000000001},"min":{"1min":0.35499999999999998,"5min":0.214,"15min":0.19800000000000001},"max":{"1min":1.569,"5min":67.206000000000003,"15min":67.206000000000003},"last":2.8479999999999999},{"interface":"front","average":{"1min":0.71899999999999997,"5min":2.923,"15min":1.948},"min":{"1min":0.309,"5min":0.224,"15min":0.224},"max":{"1min":1.099,"5min":67.132000000000005,"15min":67.132000000000005},"last":2.8140000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":8.5169999999999995,"5min":5.7839999999999998,"15min":4.0869999999999997},"min":{"1min":0.36599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":132.792,"5min":132.792,"15min":132.792},"last":2.9209999999999998},{"interface":"front","average":{"1min":8.3770000000000007,"5min":5.6539999999999999,"15min":4.048},"min":{"1min":0.26300000000000001,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":132.53299999999999,"5min":132.53299999999999,"15min":132.53299999999999},"last":0.98999999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.5840000000000001,"15min":1.5049999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":3.5939999999999999,"5min":26.068999999999999,"15min":26.068999999999999},"last":3.0990000000000002},{"interface":"front","average":{"1min":1.024,"5min":1.4139999999999999,"15min":1.546},"min":{"1min":0.311,"5min":0.31,"15min":0.31},"max":{"1min":4.1790000000000003,"5min":5.6689999999999996,"15min":12.907999999999999},"last":2.1760000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.5349999999999999,"15min":1.6020000000000001},"min":{"1min":0.48599999999999999,"5min":0.35899999999999999,"15min":0.32500000000000001},"max":{"1min":2.5019999999999998,"5min":10.586,"15min":14.170999999999999},"last":3.0800000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.506,"15min":1.6399999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.6150000000000002,"5min":26.157,"15min":26.157},"last":7.0279999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.5880000000000001,"15min":1.524},"min":{"1min":0.52000000000000002,"5min":0.38600000000000001,"15min":0.36699999999999999},"max":{"1min":2.0790000000000002,"5min":10.625,"15min":14.212999999999999},"last":2.3730000000000002},{"interface":"front","average":{"1min":1.224,"5min":1.6519999999999999,"15min":1.6479999999999999},"min":{"1min":0.47799999999999998,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":3.573,"5min":26.132000000000001,"15min":26.132000000000001},"last":3.0649999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:33:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.885,"15min":1.974},"min":{"1min":0.41699999999999998,"5min":0.41099999999999998,"15min":0.35399999999999998},"max":{"1min":4.4779999999999998,"5min":26.109000000000002,"15min":30.774999999999999},"last":2.9889999999999999},{"interface":"front","average":{"1min":1.145,"5min":1.4790000000000001,"15min":1.6679999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.2290000000000001,"5min":5.8289999999999997,"15min":30.815000000000001},"last":2.234}]}]},{"osd":0,"up_from":88,"seq":377957122302,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265080,"kb_used_data":834096,"kb_used_omap":8,"kb_used_meta":430967,"kb_avail":92480072,"statfs":{"total":95995035648,"available":94699593728,"internally_reserved":0,"allocated":854114304,"data_stored":853307155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":441310464},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.6760000000000002,"15min":2.613},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":12.991,"5min":13.598000000000001,"15min":138.31700000000001},"last":2.508},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.2429999999999999,"15min":2.3759999999999999},"min":{"1min":0.28100000000000003,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":9.7319999999999993,"5min":12.101000000000001,"15min":144.37},"last":3.8879999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":2.4929999999999999,"15min":1.9419999999999999},"min":{"1min":0.34699999999999998,"5min":0.185,"15min":0.185},"max":{"1min":0.68500000000000005,"5min":13.425000000000001,"15min":28.745000000000001},"last":3.2120000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":2.0499999999999998,"15min":1.909},"min":{"1min":0.34999999999999998,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":1.1359999999999999,"5min":16.129000000000001,"15min":28.146999999999998},"last":3.9289999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":3.1150000000000002,"15min":2.3580000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.20899999999999999},"max":{"1min":5.3289999999999997,"5min":56,"15min":59.643000000000001},"last":3.2400000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":3.032,"15min":2.1880000000000002},"min":{"1min":0.38900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9690000000000001,"5min":54.524000000000001,"15min":59.762999999999998},"last":3.956}]},{"osd":4,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.611,"15min":1.5509999999999999},"min":{"1min":0.38500000000000001,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":1.2849999999999999,"5min":8.9920000000000009,"15min":9.0890000000000004},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.966,"15min":1.8360000000000001},"min":{"1min":0.54400000000000004,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.9710000000000001,"5min":9.2319999999999993,"15min":28.629000000000001},"last":1.403}]},{"osd":5,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":2.0739999999999998,"15min":2.0630000000000002},"min":{"1min":0.70299999999999996,"5min":0.36199999999999999,"15min":0.35399999999999998},"max":{"1min":3.0059999999999998,"5min":12.356999999999999,"15min":28.574000000000002},"last":2.4409999999999998},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.873,"15min":1.8089999999999999},"min":{"1min":0.54000000000000004,"5min":0.36299999999999999,"15min":0.32600000000000001},"max":{"1min":1.613,"5min":9.8209999999999997,"15min":28.061},"last":4.0030000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.8360000000000001,"15min":1.738},"min":{"1min":0.59499999999999997,"5min":0.35299999999999998,"15min":0.34599999999999997},"max":{"1min":1.548,"5min":16.181999999999999,"15min":27.966999999999999},"last":4.4470000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.6120000000000001,"15min":1.4710000000000001},"min":{"1min":0.58399999999999996,"5min":0.39300000000000002,"15min":0.33700000000000002},"max":{"1min":1.3300000000000001,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":0.503}]},{"osd":7,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8029999999999999,"15min":1.833},"min":{"1min":0.63200000000000001,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":1.6599999999999999,"5min":7.9169999999999998,"15min":28.109999999999999},"last":4.0339999999999998},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.8380000000000001,"15min":1.863},"min":{"1min":0.41799999999999998,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":1.5209999999999999,"5min":8.9190000000000005,"15min":28.074999999999999},"last":1.3169999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795250,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089272,"kb_used_data":699056,"kb_used_omap":11,"kb_used_meta":390196,"kb_avail":92655880,"statfs":{"total":95995035648,"available":94879621120,"internally_reserved":0,"allocated":715833344,"data_stored":715023487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":399561662},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.0790000000000002,"15min":2.2749999999999999},"min":{"1min":0.35199999999999998,"5min":0.189,"15min":0.17899999999999999},"max":{"1min":0.85899999999999999,"5min":117.2,"15min":117.2},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":2.6869999999999998,"15min":2.351},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":0.76300000000000001,"5min":121.521,"15min":121.521},"last":0.93799999999999994}]},{"osd":1,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4569999999999999,"5min":2.133,"15min":1.8109999999999999},"min":{"1min":0.41899999999999998,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":60.155000000000001,"5min":60.155000000000001,"15min":60.155000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.5640000000000001,"5min":1.982,"15min":2.0219999999999998},"min":{"1min":0.35799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":62.582999999999998,"5min":62.582999999999998,"15min":62.582999999999998},"last":0.92100000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:33:18 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":5.8869999999999996,"15min":3.3439999999999999},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":6.4370000000000003,"5min":201.33600000000001,"15min":201.33600000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.0640000000000001,"5min":5.5549999999999997,"15min":3.2829999999999999},"min":{"1min":0.40500000000000003,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":3.0510000000000002,"5min":199.95099999999999,"15min":199.95099999999999},"last":0.372}]},{"osd":4,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.359,"15min":1.9850000000000001},"min":{"1min":0.624,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.093,"5min":7.5759999999999996,"15min":19.184000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.872,"5min":1.2070000000000001,"15min":1.7889999999999999},"min":{"1min":0.628,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":1.3360000000000001,"5min":6.3049999999999997,"15min":29.408000000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.335,"15min":2.3370000000000002},"min":{"1min":0.59399999999999997,"5min":0.35299999999999998,"15min":0.31},"max":{"1min":1.202,"5min":86.423000000000002,"15min":86.423000000000002},"last":1.673},{"interface":"front","average":{"1min":0.93300000000000005,"5min":2.2400000000000002,"15min":2.0179999999999998},"min":{"1min":0.58599999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.2829999999999999,"5min":86.382999999999996,"15min":86.382999999999996},"last":2.4209999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.4910000000000001,"15min":1.6419999999999999},"min":{"1min":0.70699999999999996,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":1.377,"5min":8.2959999999999994,"15min":29.504000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.4570000000000001,"15min":1.647},"min":{"1min":0.65200000000000002,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":1.1890000000000001,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.595,"15min":1.772},"min":{"1min":0.69599999999999995,"5min":0.41599999999999998,"15min":0.308},"max":{"1min":1.2849999999999999,"5min":9.2050000000000001,"15min":9.2050000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5600000000000001,"15min":2.0059999999999998},"min":{"1min":0.71399999999999997,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":1.2190000000000001,"5min":7.6920000000000002,"15min":12.051},"last":0.90000000000000002}]}]},{"osd":3,"up_from":103,"seq":442381631724,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138880,"kb_used_data":714168,"kb_used_omap":3,"kb_used_meta":424700,"kb_avail":92606272,"statfs":{"total":95995035648,"available":94828822528,"internally_reserved":0,"allocated":731308032,"data_stored":730490875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":434893345},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":2.411,"15min":2.593},"min":{"1min":0.28299999999999997,"5min":0.184,"15min":0.16900000000000001},"max":{"1min":12.952,"5min":15.285,"15min":147.988},"last":3.117},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.1720000000000002,"15min":2.4380000000000002},"min":{"1min":0.23599999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":23.646000000000001,"5min":23.646000000000001,"15min":147.94200000000001},"last":4.9589999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.4790000000000001,"15min":1.6970000000000001},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.19},"max":{"1min":8.673,"5min":47.057000000000002,"15min":47.057000000000002},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.887,"5min":2.968,"15min":2.0390000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.218},"max":{"1min":10.488,"5min":54.752000000000002,"15min":54.752000000000002},"last":5.7249999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":3.0110000000000001,"15min":2.504},"min":{"1min":0.29899999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":14.968,"5min":47.097999999999999,"15min":70.594999999999999},"last":3.1669999999999998},{"interface":"front","average":{"1min":1.3169999999999999,"5min":2.4929999999999999,"15min":2.0339999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":4.0730000000000004,"5min":46.997,"15min":62.991999999999997},"last":3.0430000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":2.641,"15min":1.964},"min":{"1min":0.35799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6470000000000002,"5min":72.653000000000006,"15min":72.653000000000006},"last":2.0059999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.444,"15min":2.0459999999999998},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":3.1779999999999999,"5min":72.706999999999994,"15min":72.706999999999994},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.8440000000000001,"15min":1.7390000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":3.1459999999999999,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":2.1200000000000001,"5min":1.962,"15min":1.8180000000000001},"min":{"1min":0.41599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.6739999999999995,"5min":9.4619999999999997,"15min":11.754},"last":2.0489999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.98,"15min":1.6890000000000001},"min":{"1min":0.55500000000000005,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":10.384,"5min":15.122999999999999,"15min":15.122999999999999},"last":3.0880000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.9670000000000001,"15min":1.649},"min":{"1min":0.377,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.2080000000000002,"5min":11.728999999999999,"15min":11.728999999999999},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:33:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.0870000000000002,"15min":1.8300000000000001},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":18.695},"last":2.0739999999999998},{"interface":"front","average":{"1min":1.7769999999999999,"5min":2.1240000000000001,"15min":1.9259999999999999},"min":{"1min":0.52200000000000002,"5min":0.43099999999999999,"15min":0.39900000000000002},"max":{"1min":10.409000000000001,"5min":13.211,"15min":13.211},"last":3.1429999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019333,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020124,"kb_used_data":722900,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92725028,"statfs":{"total":95995035648,"available":94950428672,"internally_reserved":0,"allocated":740249600,"data_stored":739700055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":2.6739999999999999,"15min":2.3700000000000001},"min":{"1min":0.58599999999999997,"5min":0.254,"15min":0.254},"max":{"1min":15.282999999999999,"5min":115.59699999999999,"15min":115.59699999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.825,"5min":2.7869999999999999,"15min":2.694},"min":{"1min":0.66400000000000003,"5min":0.28000000000000003,"15min":0.27700000000000002},"max":{"1min":15.242000000000001,"5min":115.65000000000001,"15min":115.65000000000001},"last":1.73}]},{"osd":1,"last update":"Mon Oct 9 06:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.292,"15min":2.2130000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":11.846,"5min":11.846,"15min":179.47300000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.603,"5min":1.476,"15min":2.6360000000000001},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.28499999999999998},"max":{"1min":5.1619999999999999,"5min":5.3360000000000003,"15min":179.447},"last":2.726}]},{"osd":2,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.6140000000000001,"15min":2.1309999999999998},"min":{"1min":0.57199999999999995,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":1.2689999999999999,"5min":14.705,"15min":61.783999999999999},"last":1.675},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.5760000000000001,"15min":2.2599999999999998},"min":{"1min":0.55400000000000005,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":1.3069999999999999,"5min":9.7759999999999998,"15min":61.551000000000002},"last":1.4970000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":2.1709999999999998,"15min":2.4199999999999999},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":5.548,"5min":12.081,"15min":91.361999999999995},"last":1.4510000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.9550000000000001,"15min":2.0179999999999998},"min":{"1min":0.40799999999999997,"5min":0.36699999999999999,"15min":0.313},"max":{"1min":16.257000000000001,"5min":16.257000000000001,"15min":89.676000000000002},"last":0.58099999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:33:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.85199999999999998,"15min":1.278},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1830000000000001,"5min":11.737,"15min":12.242000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.135,"5min":1.1140000000000001,"15min":1.6950000000000001},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.26500000000000001},"max":{"1min":6.7539999999999996,"5min":6.7539999999999996,"15min":20.113},"last":7.7649999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.288,"15min":1.5820000000000001},"min":{"1min":0.54700000000000004,"5min":0.35399999999999998,"15min":0.29499999999999998},"max":{"1min":1.609,"5min":8.1110000000000007,"15min":18.010000000000002},"last":4.7069999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.2809999999999999,"15min":1.4319999999999999},"min":{"1min":0.39100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4199999999999999,"5min":11.782,"15min":13.913},"last":0.69499999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.4419999999999999,"15min":1.776},"min":{"1min":0.73399999999999999,"5min":0.42399999999999999,"15min":0.371},"max":{"1min":2.6480000000000001,"5min":12.779,"15min":15.294},"last":1.762},{"interface":"front","average":{"1min":1.006,"5min":1.226,"15min":1.7470000000000001},"min":{"1min":0.432,"5min":0.317,"15min":0.317},"max":{"1min":1.6399999999999999,"5min":9.1370000000000005,"15min":14.896000000000001},"last":1.5329999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823100,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1437448,"kb_used_data":1132032,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92307704,"statfs":{"total":95995035648,"available":94523088896,"internally_reserved":0,"allocated":1159200768,"data_stored":1158666836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":2.9169999999999998,"15min":2.6989999999999998},"min":{"1min":0.56999999999999995,"5min":0.254,"15min":0.247},"max":{"1min":1.0800000000000001,"5min":83.197000000000003,"15min":83.197000000000003},"last":4.9900000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.7360000000000002,"15min":2.4209999999999998},"min":{"1min":0.56899999999999995,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":1.3160000000000001,"5min":83.563999999999993,"15min":83.563999999999993},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:33:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.411,"15min":2.4209999999999998},"min":{"1min":0.45700000000000002,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":4.8289999999999997,"5min":5.9299999999999997,"15min":161.71199999999999},"last":0.308},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.5429999999999999,"15min":2.653},"min":{"1min":0.38400000000000001,"5min":0.31,"15min":0.29399999999999998},"max":{"1min":4.4219999999999997,"5min":9.0589999999999993,"15min":162.50200000000001},"last":0.63}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":2.0979999999999999,"15min":2.2890000000000001},"min":{"1min":0.67000000000000004,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":1.1970000000000001,"5min":54.981999999999999,"15min":54.981999999999999},"last":5.0380000000000003},{"interface":"front","average":{"1min":1.052,"5min":2.2010000000000001,"15min":2.2389999999999999},"min":{"1min":0.64500000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":2.149,"5min":55.008000000000003,"15min":59.082999999999998},"last":0.33000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":5.3209999999999997,"5min":4.1379999999999999,"15min":2.8740000000000001},"min":{"1min":0.438,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":84.174000000000007,"5min":84.174000000000007,"15min":84.174000000000007},"last":5.0670000000000002},{"interface":"front","average":{"1min":6.0369999999999999,"5min":4.1120000000000001,"15min":2.7120000000000002},"min":{"1min":0.41399999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":83.114999999999995,"5min":83.114999999999995,"15min":83.114999999999995},"last":0.35299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:32 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.252,"15min":1.8839999999999999},"min":{"1min":0.41699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.4100000000000001,"5min":8.8879999999999999,"15min":17.667999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.137,"15min":1.5900000000000001},"min":{"1min":0.57399999999999995,"5min":0.308,"15min":0.308},"max":{"1min":2.1909999999999998,"5min":5.3040000000000003,"15min":16.181999999999999},"last":5.0919999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.2609999999999999,"15min":1.702},"min":{"1min":0.442,"5min":0.37,"15min":0.35699999999999998},"max":{"1min":3.5699999999999998,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.008,"5min":1.264,"15min":1.6160000000000001},"min":{"1min":0.47799999999999998,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":1.9350000000000001,"5min":9.3450000000000006,"15min":12.365},"last":5.1180000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:33:50 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.115,"15min":1.708},"min":{"1min":0.99299999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.4590000000000001,"5min":5.7039999999999997,"15min":13.047000000000001},"last":5.7190000000000003},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.2350000000000001,"15min":1.526},"min":{"1min":0.91900000000000004,"5min":0.41599999999999998,"15min":0.36799999999999999},"max":{"1min":1.4379999999999999,"5min":5.5919999999999996,"15min":11.981},"last":0.374}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":864813056,"data_stored":864813056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":925040640,"data_stored":925040640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":753467392,"data_stored":753467392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":767033344,"data_stored":767033344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":741146624,"data_stored":741146624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1194131456,"data_stored":1194131456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":753860608,"data_stored":753860608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":751370240,"data_stored":751370240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:19.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:19.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:19.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:19.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:19.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:19.578 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:19 smithi155 bash[59495]: audit 2023-10-09T06:34:18.588593+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:19.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:19.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:19 smithi125 bash[77299]: audit 2023-10-09T06:34:18.588593+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:19.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:19 smithi125 bash[80002]: audit 2023-10-09T06:34:18.588593+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:19.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:19.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:19.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 44117 44101 119.824 93.125 0.00720186 0.00833954 2023-10-09T06:34:19.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:19.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:19.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:19.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:20.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:20.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:20.233 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:20 smithi155 bash[59495]: cluster 2023-10-09T06:34:19.123278+0000 mgr.y (mgr.25723) 886 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:34:20.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:20.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:20.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:20.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:20.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:20.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:20.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:20.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:20.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:20 smithi125 bash[77299]: cluster 2023-10-09T06:34:19.123278+0000 mgr.y (mgr.25723) 886 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:34:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:20 smithi125 bash[80002]: cluster 2023-10-09T06:34:19.123278+0000 mgr.y (mgr.25723) 886 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:34:20.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:20.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:20.783 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 45520 45504 118.484 87.6875 0.0090169 0.0084336 2023-10-09T06:34:20.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:20.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:20.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:20.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:20.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:21.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:21.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:21.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:21.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:21.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:21.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:21.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:21.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:21.448 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:21.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:21.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:21.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:21.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:21.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:21] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:34:21.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:21.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:21.788 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 46955 46939 117.312 89.6875 0.00660074 0.00851729 2023-10-09T06:34:21.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:21.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:21.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:22.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:22.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:22.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:22.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:22.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.280 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:22 smithi125 bash[77299]: cluster 2023-10-09T06:34:21.123936+0000 mgr.y (mgr.25723) 887 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:34:22.281 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:22 smithi125 bash[80002]: cluster 2023-10-09T06:34:21.123936+0000 mgr.y (mgr.25723) 887 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:34:22.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:22.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:22.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:22.561 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:22 smithi155 bash[59495]: cluster 2023-10-09T06:34:21.123936+0000 mgr.y (mgr.25723) 887 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:34:22.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:22.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:22.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:22.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:22.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:22.792 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 48271 48255 115.944 82.25 0.00469474 0.00861876 2023-10-09T06:34:22.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:22.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:22.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:22.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:22.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:23.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:23.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:23.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:23.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:23.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:23.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:23.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:23.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:23.747 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:23.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.798 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 49601 49585 114.713 83.125 0.0176215 0.00871176 2023-10-09T06:34:23.803 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:23.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:23.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:24.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:24.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:24.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:24.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:24.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:24.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:24.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:24.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:24.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:24.578 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:24 smithi155 bash[59495]: cluster 2023-10-09T06:34:23.124548+0000 mgr.y (mgr.25723) 888 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:34:24.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:24.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:24 smithi125 bash[77299]: cluster 2023-10-09T06:34:23.124548+0000 mgr.y (mgr.25723) 888 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:34:24.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:24 smithi125 bash[80002]: cluster 2023-10-09T06:34:23.124548+0000 mgr.y (mgr.25723) 888 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-09T06:34:24.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:24.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:24.796 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 50507 50491 112.639 56.625 0.014641 0.00887122 2023-10-09T06:34:24.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:24.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:24.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:24.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:25.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:25.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:25.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:25.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:25.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:25.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:25.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:25.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:25.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:25.796 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 51812 51796 111.567 81.5625 0.0151573 0.0089564 2023-10-09T06:34:25.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:25.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:25.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:25.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:26.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:26.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:26.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:26.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:26.311 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:26.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:26 smithi155 bash[59495]: cluster 2023-10-09T06:34:25.125650+0000 mgr.y (mgr.25723) 889 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T06:34:26.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:26.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:26 smithi125 bash[77299]: cluster 2023-10-09T06:34:25.125650+0000 mgr.y (mgr.25723) 889 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T06:34:26.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:26 smithi125 bash[80002]: cluster 2023-10-09T06:34:25.125650+0000 mgr.y (mgr.25723) 889 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2023-10-09T06:34:26.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:26.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:26.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:26.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:26.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:26.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:26.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:26.796 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 53042 53026 110.411 76.875 0.018999 0.00905155 2023-10-09T06:34:26.805 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:26.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:26.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:26.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:26.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:27.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:27.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:27.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:27.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:27.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:27.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:27.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:27.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:27.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:27.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:27.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 54313 54297 109.41 79.4375 0.00691691 0.00913396 2023-10-09T06:34:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:27.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:27.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:28.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:28.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:28.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:28.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:28 smithi125 bash[80002]: cluster 2023-10-09T06:34:27.126225+0000 mgr.y (mgr.25723) 890 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:34:28.323 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:28 smithi125 bash[80002]: audit 2023-10-09T06:34:27.874858+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:28.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:28 smithi125 bash[77299]: cluster 2023-10-09T06:34:27.126225+0000 mgr.y (mgr.25723) 890 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:34:28.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:28 smithi125 bash[77299]: audit 2023-10-09T06:34:27.874858+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:28.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:28.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:28.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:28 smithi155 bash[59495]: cluster 2023-10-09T06:34:27.126225+0000 mgr.y (mgr.25723) 890 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:34:28.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:28 smithi155 bash[59495]: audit 2023-10-09T06:34:27.874858+0000 mon.a (mon.0) 1090 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:28.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:28.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:28.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:28.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:28.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 55486 55470 108.282 73.3125 0.0130649 0.00922975 2023-10-09T06:34:28.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:28.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:28.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:28.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:28.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:29.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:29.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:29.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:29.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:29.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:29.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:29.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[80002]: audit 2023-10-09T06:34:28.360967+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[80002]: audit 2023-10-09T06:34:28.374634+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.364 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[80002]: cluster 2023-10-09T06:34:29.127072+0000 mgr.y (mgr.25723) 891 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:34:29.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:29.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:29.499 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:29 smithi155 bash[59495]: audit 2023-10-09T06:34:28.360967+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:29 smithi155 bash[59495]: audit 2023-10-09T06:34:28.374634+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.500 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:29 smithi155 bash[59495]: cluster 2023-10-09T06:34:29.127072+0000 mgr.y (mgr.25723) 891 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:34:29.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:29.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:29.643 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:29.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[77299]: audit 2023-10-09T06:34:28.360967+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[77299]: audit 2023-10-09T06:34:28.374634+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:29.677 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:29 smithi125 bash[77299]: cluster 2023-10-09T06:34:29.127072+0000 mgr.y (mgr.25723) 891 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:34:29.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:29.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:29.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 56491 56475 106.905 62.8125 0.00774662 0.00934886 2023-10-09T06:34:29.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:29.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:29.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:30.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:30.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:30.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:30.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:30.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:30.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:30.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:30.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:30.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:30.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:30.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:30.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:30.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:30.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 57672 57656 105.932 73.8125 0.0129208 0.00943451 2023-10-09T06:34:30.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:30.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:30.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:30.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:30.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:31.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.112 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:31.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:31.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:31.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:31.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:31] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:34:31.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:31.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:31.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:31.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:31.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:31.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:31.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 58721 58705 104.779 65.5625 0.00770057 0.00953889 2023-10-09T06:34:31.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:31.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:31.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:32.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:32.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:32.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:32.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:32.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:32.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:32.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:32.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:32 smithi125 bash[77299]: cluster 2023-10-09T06:34:31.127716+0000 mgr.y (mgr.25723) 892 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:34:32.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:32 smithi125 bash[80002]: cluster 2023-10-09T06:34:31.127716+0000 mgr.y (mgr.25723) 892 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:34:32.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:32.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:32.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:32.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:32 smithi155 bash[59495]: cluster 2023-10-09T06:34:31.127716+0000 mgr.y (mgr.25723) 892 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:34:32.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:32.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:32.798 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 60007 59991 104.101 80.375 0.0104359 0.00960055 2023-10-09T06:34:32.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:32.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:32.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:32.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:32.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:33.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:33.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:33.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:33.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:33.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:33.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:33.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:33.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:33.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:33.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:33.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:33.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:33.797 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 61176 61160 103.262 73.0625 0.00986476 0.00967907 2023-10-09T06:34:33.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:33.902 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:33.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:34.006 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:34.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:34.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:34.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:34.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.446 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:34.451 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:34 smithi155 bash[59495]: cluster 2023-10-09T06:34:33.128316+0000 mgr.y (mgr.25723) 893 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:34:34.451 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:34 smithi155 bash[59495]: audit 2023-10-09T06:34:33.589105+0000 mon.a (mon.0) 1093 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:34.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:34.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:34 smithi125 bash[77299]: cluster 2023-10-09T06:34:33.128316+0000 mgr.y (mgr.25723) 893 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:34:34.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:34 smithi125 bash[77299]: audit 2023-10-09T06:34:33.589105+0000 mon.a (mon.0) 1093 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:34.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:34 smithi125 bash[80002]: cluster 2023-10-09T06:34:33.128316+0000 mgr.y (mgr.25723) 893 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:34:34.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:34 smithi125 bash[80002]: audit 2023-10-09T06:34:33.589105+0000 mon.a (mon.0) 1093 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:34.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:34.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:34.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 62178 62162 102.186 62.625 0.0157333 0.00978017 2023-10-09T06:34:34.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:34.948 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:34.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:35.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:35.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:35.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:35.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:35.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:35.484 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:35.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:35.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:35.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:35.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:35.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:35.800 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 63070 63054 100.996 55.75 0.0137794 0.00989589 2023-10-09T06:34:35.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:35.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:35.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:36.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:36.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:36.039 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:35 smithi155 bash[59495]: audit 2023-10-09T06:34:34.833667+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.039 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:35 smithi155 bash[59495]: audit 2023-10-09T06:34:34.845523+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.039 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:35 smithi155 bash[59495]: cluster 2023-10-09T06:34:35.129245+0000 mgr.y (mgr.25723) 894 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:34:36.040 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:35 smithi155 bash[59495]: audit 2023-10-09T06:34:35.599147+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.040 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:35 smithi155 bash[59495]: audit 2023-10-09T06:34:35.612975+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:36.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[77299]: audit 2023-10-09T06:34:34.833667+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[77299]: audit 2023-10-09T06:34:34.845523+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[77299]: cluster 2023-10-09T06:34:35.129245+0000 mgr.y (mgr.25723) 894 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[77299]: audit 2023-10-09T06:34:35.599147+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[77299]: audit 2023-10-09T06:34:35.612975+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[80002]: audit 2023-10-09T06:34:34.833667+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[80002]: audit 2023-10-09T06:34:34.845523+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[80002]: cluster 2023-10-09T06:34:35.129245+0000 mgr.y (mgr.25723) 894 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:34:36.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[80002]: audit 2023-10-09T06:34:35.599147+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:35 smithi125 bash[80002]: audit 2023-10-09T06:34:35.612975+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.142 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:36.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:36.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:36.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:36.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:36.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:36.549 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:36.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:36.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:36.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:36.800 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:34:36.794703+0000 min lat: 0.00131043 max lat: 0.352992 avg lat: 0.00997055 2023-10-09T06:34:36.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:34:36.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 64205 64189 100.245 70.9375 0.017358 0.00997055 2023-10-09T06:34:36.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:36.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:36.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:36.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[77299]: audit 2023-10-09T06:34:36.192436+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:36.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[77299]: audit 2023-10-09T06:34:36.195565+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:36.961 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[77299]: audit 2023-10-09T06:34:36.205946+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.962 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[80002]: audit 2023-10-09T06:34:36.192436+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:36.962 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[80002]: audit 2023-10-09T06:34:36.195565+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:36.962 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:36 smithi125 bash[80002]: audit 2023-10-09T06:34:36.205946+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:36.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:36.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:37.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:37.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:36 smithi155 bash[59495]: audit 2023-10-09T06:34:36.192436+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:36 smithi155 bash[59495]: audit 2023-10-09T06:34:36.195565+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:37.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:36 smithi155 bash[59495]: audit 2023-10-09T06:34:36.205946+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:34:37.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:37.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:37.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:37.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:37.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:37.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:37.621 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:37.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:37.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:37.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:37.800 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 65215 65199 99.3394 63.125 0.0170399 0.0100613 2023-10-09T06:34:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:37 smithi155 bash[59495]: cluster 2023-10-09T06:34:37.129741+0000 mgr.y (mgr.25723) 895 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T06:34:37.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:37.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:37.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:38.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:38.032 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:37 smithi125 bash[77299]: cluster 2023-10-09T06:34:37.129741+0000 mgr.y (mgr.25723) 895 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T06:34:38.033 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:37 smithi125 bash[80002]: cluster 2023-10-09T06:34:37.129741+0000 mgr.y (mgr.25723) 895 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-10-09T06:34:38.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:38.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:38.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:38.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:38.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:38.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:38.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:38.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:38.583 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:38.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:38.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:38.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:38.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 66237 66221 98.4952 63.875 0.00822669 0.0101447 2023-10-09T06:34:38.809 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:38.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:38.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:38.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:38.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:39.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:39.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:39.206 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:39.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:39.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:39.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:39.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:39.703 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:34:39.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:39.757 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:34:39.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:39.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 67333 67317 97.7978 68.5 0.00416088 0.0102172 2023-10-09T06:34:39.803 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:34:39.837 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:34:39.871 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:34:39.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:39.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:39.920 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:34:39.929 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:34:39.958 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:34:39.985 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:34:39.985 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:34:39.985 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:34:39.985 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:27634 7.2 7:4bf4d30c:::benchmark_data_smithi125_153312_object27633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.431428+0000", 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.53939299500001, 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.254933617, 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:39.986 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27634 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431428+0000", 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431428+0000", 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:39.987 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431427+0000", 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431459+0000", 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1770000000000002e-05 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.988 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431482+0000", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3221999999999999e-05 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.431489+0000", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9829999999999999e-06 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.433326+0000", 2023-10-09T06:34:39.989 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001837193 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.433445+0000", 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011895099999999999 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.433576+0000", 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013126300000000001 2023-10-09T06:34:39.990 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435897+0000", 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0023203450000000001 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.439474+0000", 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035772719999999998 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.991 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445275+0000", 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0058014099999999999 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445316+0000", 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0111999999999998e-05 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.992 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686362+0000", 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24104611100000001 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:27623 7.f e173/165 7:f3cf21f8:::benchmark_data_smithi125_153312_object27622:head v 173'1733, mlcod=173'1729)", 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.435642+0000", 2023-10-09T06:34:39.993 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.53517908600003, 2023-10-09T06:34:39.994 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.25066486500000001, 2023-10-09T06:34:39.994 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435642+0000", 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435642+0000", 2023-10-09T06:34:39.995 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:39.996 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.996 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.996 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435641+0000", 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435761+0000", 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011953199999999999 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.997 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435840+0000", 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9269000000000003e-05 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.435845+0000", 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1919999999999996e-06 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:39.998 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:39.999 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:39.999 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.443521+0000", 2023-10-09T06:34:39.999 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076761090000000004 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.443541+0000", 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9593e-05 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.000 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.446083+0000", 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025415989999999999 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686307+0000", 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24022416099999999 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.001 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:27641 7.3 7:cb846a5a:::benchmark_data_smithi125_153312_object27640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.437828+0000", 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.532993385, 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24860178299999999, 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27641 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.002 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437828+0000", 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437828+0000", 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.003 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437826+0000", 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437846+0000", 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.9828999999999999e-05 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.004 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437855+0000", 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1390000000000001e-06 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.437862+0000", 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3450000000000002e-06 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.005 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.438037+0000", 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017591300000000001 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.439645+0000", 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0016071359999999999 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.439850+0000", 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020592000000000001 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441394+0000", 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0015430559999999999 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.443178+0000", 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0017844289999999999 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.007 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686211+0000", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24303348899999999 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686337+0000", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126029 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686429+0000", 2023-10-09T06:34:40.008 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1964000000000002e-05 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:27649 7.3 7:c7c78c24:::benchmark_data_smithi125_153312_object27648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.440609+0000", 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.53021193500001, 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24643806300000001, 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.009 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27649 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440609+0000", 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.010 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440609+0000", 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440608+0000", 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.011 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440626+0000", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8119999999999999e-05 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440637+0000", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0658e-05 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.440643+0000", 2023-10-09T06:34:40.012 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0329999999999997e-06 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441421+0000", 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00077779100000000001 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441568+0000", 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014778999999999999 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.013 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441720+0000", 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000151812 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.443194+0000", 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0014740860000000001 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.014 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686560+0000", 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24336606899999999 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686838+0000", 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027775199999999998 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686960+0000", 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012213799999999999 2023-10-09T06:34:40.015 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.687047+0000", 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6939000000000002e-05 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:27644 7.f e173/165 7:f1e53ae6:::benchmark_data_smithi125_153312_object27643:head v 173'1734, mlcod=173'1730)", 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.441413+0000", 2023-10-09T06:34:40.016 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52940813700002, 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.25415166300000003, 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441413+0000", 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.017 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441413+0000", 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441412+0000", 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441435+0000", 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3252999999999999e-05 2023-10-09T06:34:40.018 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441446+0000", 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0919e-05 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.441454+0000", 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6240000000000002e-06 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.019 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686424+0000", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.244970154 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686478+0000", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4070999999999998e-05 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.695384+0000", 2023-10-09T06:34:40.020 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0089056250000000003 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.695565+0000", 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000180786 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:27646 7.6 e173/165 7:60f6048c:::benchmark_data_smithi125_153312_object27645:head v 173'1708, mlcod=173'1706)", 2023-10-09T06:34:40.021 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.444770+0000", 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52605109799998, 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24142396599999999, 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444770+0000", 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.022 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444770+0000", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444769+0000", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444797+0000", 2023-10-09T06:34:40.023 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7855e-05 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444808+0000", 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0845e-05 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444815+0000", 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.2620000000000001e-06 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.024 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444857+0000", 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2057000000000001e-05 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444892+0000", 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4776e-05 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.025 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686097+0000", 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.241204643 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686194+0000", 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7097999999999998e-05 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.026 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.45170.0:27641 7.3 e173/165 ondisk, result = 0)", 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.444905+0000", 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52591628699997, 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24154926500000001, 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444905+0000", 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.027 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444905+0000", 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444904+0000", 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.028 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444908+0000", 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1910000000000002e-06 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444917+0000", 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6589999999999996e-06 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.029 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444923+0000", 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2339999999999999e-06 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686171+0000", 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24124812000000001 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686195+0000", 2023-10-09T06:34:40.030 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3269999999999999e-05 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686454+0000", 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025930300000000001 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.031 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.45170.0:27634 7.2 e173/165 ondisk, result = 0)", 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.444959+0000", 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52586238100002, 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24141515299999999, 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444959+0000", 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.032 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444959+0000", 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444958+0000", 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.033 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444961+0000", 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.091e-06 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444966+0000", 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7239999999999997e-06 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.034 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444971+0000", 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4909999999999999e-06 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445264+0000", 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029307699999999999 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.035 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445270+0000", 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4609999999999996e-06 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686374+0000", 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.241103553 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:27650 7.0 e173/165 7:0fa81ccf:::benchmark_data_smithi125_153312_object27649:head v 173'1737, mlcod=173'1735)", 2023-10-09T06:34:40.036 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.444994+0000", 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.525827026, 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24128348999999999, 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444994+0000", 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.037 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444994+0000", 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.444994+0000", 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.038 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445031+0000", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7877999999999999e-05 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445040+0000", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2320000000000001e-06 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445045+0000", 2023-10-09T06:34:40.039 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6450000000000002e-06 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445104+0000", 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8486000000000002e-05 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445124+0000", 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0276e-05 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.040 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686190+0000", 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.241066157 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686278+0000", 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7284e-05 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.041 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.45170.0:27649 7.3 e173/165 ondisk, result = 0)", 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.445245+0000", 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52557578199998, 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24132300700000001, 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445245+0000", 2023-10-09T06:34:40.042 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445245+0000", 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445245+0000", 2023-10-09T06:34:40.043 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445252+0000", 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7950000000000008e-06 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445261+0000", 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7690000000000002e-06 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.044 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445267+0000", 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7930000000000003e-06 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686528+0000", 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24126136400000001 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.045 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686546+0000", 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7842999999999998e-05 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686568+0000", 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2079999999999999e-05 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.046 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:27655 7.2 7:4c5d431d:::benchmark_data_smithi125_153312_object27654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.445295+0000", 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.525526243, 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24790405200000001, 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 27655 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.047 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445295+0000", 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445295+0000", 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.048 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445293+0000", 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.445315+0000", 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1654999999999998e-05 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686045+0000", 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.240730098 2023-10-09T06:34:40.049 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686068+0000", 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.245e-05 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686398+0000", 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00033022299999999999 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.050 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686602+0000", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020366399999999999 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.687857+0000", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012553359999999999 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.692759+0000", 2023-10-09T06:34:40.051 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049024919999999996 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.692768+0000", 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2609999999999993e-06 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.693012+0000", 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024484600000000001 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.052 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.693086+0000", 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3226999999999995e-05 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.693199+0000", 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011313 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.053 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:27656 7.0 e173/165 7:03523190:::benchmark_data_smithi125_153312_object27655:head v 173'1738, mlcod=173'1735)", 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:28:51.446543+0000", 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 348.52427792499998, 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.243597129, 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.054 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.446543+0000", 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.446543+0000", 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.446542+0000", 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.055 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.446574+0000", 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1804000000000002e-05 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686095+0000", 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.23952114499999999 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.056 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686107+0000", 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1275e-05 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686350+0000", 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024297399999999999 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.686400+0000", 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0336000000000001e-05 2023-10-09T06:34:40.057 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.690089+0000", 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0036886649999999998 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:28:51.690140+0000", 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1585999999999997e-05 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.058 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:71442 7.2 7:4caf1fb4:::benchmark_data_smithi125_153312_object71441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:28.021699+0000", 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 311.94912221499999, 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.239513, 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.059 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 71442 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021699+0000", 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.060 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021699+0000", 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021697+0000", 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.061 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021721+0000", 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4077e-05 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021734+0000", 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2346e-05 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.062 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.021742+0000", 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8390000000000007e-06 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022738+0000", 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00099695500000000011 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022889+0000", 2023-10-09T06:34:40.063 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015055299999999999 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.244098+0000", 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22120879600000001 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.249015+0000", 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049173580000000001 2023-10-09T06:34:40.064 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.257605+0000", 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0085902470000000005 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261122+0000", 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035168119999999998 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.065 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261174+0000", 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1569e-05 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261212+0000", 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8006999999999997e-05 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.066 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:71445 7.a 7:5a90c64a:::benchmark_data_smithi125_153312_object71444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:29:28.022681+0000", 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 311.94814017700003, 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.238611559, 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.067 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 71445 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022681+0000", 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.068 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022681+0000", 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022680+0000", 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.069 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022715+0000", 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5012999999999998e-05 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022725+0000", 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0161e-05 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022732+0000", 2023-10-09T06:34:40.070 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9369999999999998e-06 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022760+0000", 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8228000000000001e-05 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.022899+0000", 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013862100000000001 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.071 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.244137+0000", 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.221237814 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.257663+0000", 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.013526521 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.072 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.257715+0000", 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2024999999999997e-05 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261222+0000", 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035069620000000002 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261258+0000", 2023-10-09T06:34:40.073 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5524000000000001e-05 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:29:28.261292+0000", 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4620999999999999e-05 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.074 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:198341 7.c 7:312a362a:::benchmark_data_smithi125_153312_object93654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:32:24.777338+0000", 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 135.19348267000001, 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24607142500000001, 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 198341 2023-10-09T06:34:40.075 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777338+0000", 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777338+0000", 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.076 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777337+0000", 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777344+0000", 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4710000000000004e-06 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.077 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777356+0000", 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2539e-05 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777364+0000", 2023-10-09T06:34:40.078 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4379999999999998e-06 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777429+0000", 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5696000000000003e-05 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777603+0000", 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017333 2023-10-09T06:34:40.079 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.777768+0000", 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016582200000000001 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.785837+0000", 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080687190000000002 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.080 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.015462+0000", 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22962458699999999 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023199+0000", 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0077368860000000001 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.081 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023262+0000", 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3745999999999996e-05 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023410+0000", 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014738699999999999 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.082 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45170.0:198347 7.c 7:3dc4aaf9:::benchmark_data_smithi125_153312_object93660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:32:24.782253+0000", 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 135.18856770299999, 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24128459999999999, 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45170", 2023-10-09T06:34:40.083 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 198347 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782253+0000", 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.084 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782253+0000", 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782252+0000", 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782259+0000", 2023-10-09T06:34:40.085 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5950000000000004e-06 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782270+0000", 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1419e-05 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782278+0000", 2023-10-09T06:34:40.086 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5349999999999999e-06 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.785856+0000", 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0035781229999999999 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.785939+0000", 2023-10-09T06:34:40.087 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3646999999999994e-05 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.005865+0000", 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.219925597 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.015497+0000", 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0096318489999999996 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.088 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023177+0000", 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076804610000000004 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023435+0000", 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025797199999999998 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.089 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023507+0000", 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1513000000000005e-05 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.023538+0000", 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1229000000000002e-05 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.090 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45170.0:198340 7.f e185/165 7:fd69d626:::benchmark_data_smithi125_153312_object93653:head v 185'12311, mlcod=185'12307)", 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:32:24.782983+0000", 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 135.18783848699999, 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.24109715200000001, 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.091 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782983+0000", 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782983+0000", 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.092 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782982+0000", 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782987+0000", 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6879999999999998e-06 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.782997+0000", 2023-10-09T06:34:40.093 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0234999999999999e-05 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:24.783003+0000", 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2439999999999998e-06 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.094 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.006283+0000", 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.22328000000000001 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.006308+0000", 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4774e-05 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.024051+0000", 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017743489000000001 2023-10-09T06:34:40.095 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:32:25.024080+0000", 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8364000000000002e-05 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18138 8.b 8:de1a6651:::benchmark_data_smithi125_176369_object18137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:34:40.096 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 35.812876428000003, 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.347658947, 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18138 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.097 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.098 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157943+0000", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157969+0000", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6078999999999999e-05 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157986+0000", 2023-10-09T06:34:40.099 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7246999999999999e-05 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157995+0000", 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6240000000000008e-06 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158073+0000", 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8320999999999998e-05 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.100 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158244+0000", 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017047899999999999 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158441+0000", 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196925 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.101 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.159608+0000", 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011669180000000001 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.161628+0000", 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020198740000000001 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.102 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505490+0000", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34386208600000001 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505553+0000", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3676000000000005e-05 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505604+0000", 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0297999999999999e-05 2023-10-09T06:34:40.103 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18162 8.0 8:07fcf932:::benchmark_data_smithi125_176369_object18161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 35.800295787000003, 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33879309499999999, 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.104 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18162 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.105 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170523+0000", 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170552+0000", 2023-10-09T06:34:40.106 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8835000000000001e-05 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170566+0000", 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.452e-05 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170575+0000", 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0750000000000004e-06 2023-10-09T06:34:40.107 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170626+0000", 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.092e-05 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170820+0000", 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000193764 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:34:40.108 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170967+0000", 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000147372 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.172088+0000", 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001120881 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.109 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.172269+0000", 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018081400000000001 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509161+0000", 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.336891579 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509250+0000", 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9395999999999996e-05 2023-10-09T06:34:40.110 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509318+0000", 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8211000000000001e-05 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18175 8.f 8:ff073d6a:::benchmark_data_smithi125_176369_object18174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:34:40.111 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 35.790786283000003, 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33486070699999998, 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18175 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:34:40.112 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180032+0000", 2023-10-09T06:34:40.113 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180068+0000", 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5939999999999998e-05 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180092+0000", 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3649e-05 2023-10-09T06:34:40.114 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180105+0000", 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3287e-05 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180162+0000", 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6471999999999997e-05 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.115 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180321+0000", 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000159159 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180473+0000", 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000152292 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.116 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.181790+0000", 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001316397 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.181950+0000", 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016058199999999999 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.117 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514801+0000", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33285042599999998 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514855+0000", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4542000000000001e-05 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514895+0000", 2023-10-09T06:34:40.118 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0299999999999997e-05 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:34:40.119 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:34:40.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:40.120 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.121 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:34:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:39 smithi155 bash[59495]: audit 2023-10-09T06:34:39.036671+0000 mon.a (mon.0) 1101 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:39 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.130226+0000 mgr.y (mgr.25723) 896 : cluster 0 pgmap v906: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:34:40.129 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:39 smithi155 bash[59495]: audit 2023-10-09T06:34:39.645205+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.125:0/2371883388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[77299]: audit 2023-10-09T06:34:39.036671+0000 mon.a (mon.0) 1101 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.130226+0000 mgr.y (mgr.25723) 896 : cluster 0 pgmap v906: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:34:40.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[77299]: audit 2023-10-09T06:34:39.645205+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.125:0/2371883388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:40.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[80002]: audit 2023-10-09T06:34:39.036671+0000 mon.a (mon.0) 1101 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.130226+0000 mgr.y (mgr.25723) 896 : cluster 0 pgmap v906: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:34:40.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:39 smithi125 bash[80002]: audit 2023-10-09T06:34:39.645205+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.125:0/2371883388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:40.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:40.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:40.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:40.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:40.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:40.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:40.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:40.652 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:34:40.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:40.661 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:34:40.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:40.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:40.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 68348 68332 97.017 63.4375 0.00665104 0.0103024 2023-10-09T06:34:40.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:40.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:40.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:40.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:41.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:41.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.905192+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:41.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: audit 2023-10-09T06:34:39.917398+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:41.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.917443+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:34:41.070 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.922688+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.923048+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.071 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:40 smithi155 bash[59495]: cluster 2023-10-09T06:34:39.928677+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:41.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:41.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.905192+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:41.215 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: audit 2023-10-09T06:34:39.917398+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.917443+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.922688+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.923048+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[77299]: cluster 2023-10-09T06:34:39.928677+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:41] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:34:41.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.905192+0000 mon.a (mon.0) 1103 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:41.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: audit 2023-10-09T06:34:39.917398+0000 mon.a (mon.0) 1104 : audit 1 from='client.? 172.21.15.125:0/733840939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:41.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.917443+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:34:41.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.922688+0000 osd.6 (osd.6) 135 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.923048+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:40 smithi125 bash[80002]: cluster 2023-10-09T06:34:39.928677+0000 osd.4 (osd.4) 125 : cluster 3 failed to encode map e198 with expected crc 2023-10-09T06:34:41.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:41.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:41.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:41.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:41.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:41.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:41.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:41.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:41.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:41.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:41.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 69502 69486 96.4639 72.125 0.012515 0.0103617 2023-10-09T06:34:41.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:41.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:41.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:41.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:41.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:42.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:42.035 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:41 smithi125 bash[77299]: audit 2023-10-09T06:34:40.624150+0000 mgr.y (mgr.25723) 897 : audit 0 from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:42.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:41 smithi125 bash[77299]: cluster 2023-10-09T06:34:41.130721+0000 mgr.y (mgr.25723) 898 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:34:42.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:41 smithi125 bash[80002]: audit 2023-10-09T06:34:40.624150+0000 mgr.y (mgr.25723) 897 : audit 0 from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:42.036 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:41 smithi125 bash[80002]: cluster 2023-10-09T06:34:41.130721+0000 mgr.y (mgr.25723) 898 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:34:42.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:42.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:41 smithi155 bash[59495]: audit 2023-10-09T06:34:40.624150+0000 mgr.y (mgr.25723) 897 : audit 0 from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:41 smithi155 bash[59495]: cluster 2023-10-09T06:34:41.130721+0000 mgr.y (mgr.25723) 898 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:34:42.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:42.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:42.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:42.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:42.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:42.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:42.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:42.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:42.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:42.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:42.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 70476 70460 95.6904 60.875 0.0196843 0.0104443 2023-10-09T06:34:42.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:42.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:42.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:43.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:43.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:43.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:43.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:43.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:43.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:43.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:43.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:43.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:43.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:43.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 71440 71424 94.9365 60.25 0.0158767 0.010528 2023-10-09T06:34:43.828 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:43.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:43.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:43.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:44.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:44.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:44.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:44.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:44.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:44.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:44.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:44 smithi155 bash[59495]: cluster 2023-10-09T06:34:43.131339+0000 mgr.y (mgr.25723) 899 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:44.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:44.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:44.479 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:44 smithi125 bash[77299]: cluster 2023-10-09T06:34:43.131339+0000 mgr.y (mgr.25723) 899 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:44.479 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:44 smithi125 bash[80002]: cluster 2023-10-09T06:34:43.131339+0000 mgr.y (mgr.25723) 899 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:44.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:44.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:44.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:44.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:44.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:44.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 72434 72418 94.2531 62.125 0.0124155 0.0106038 2023-10-09T06:34:44.876 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:44.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:44.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:45.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:45.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:45.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:34:45.139 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":907,"stamp":"2023-10-09T06:34:39.914496+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4297791411,"num_objects":65534,"num_object_clones":0,"num_object_copies":196602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65534,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":66612,"num_write_kb":4207543,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66676,"ondisk_log_size":66676,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15412128,"kb_used_data":12445600,"kb_used_omap":30,"kb_used_meta":2966433,"kb_avail":734549088,"statfs":{"total":767960285184,"available":752178266112,"internally_reserved":0,"allocated":12744294400,"data_stored":12738846101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":3037627503},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":839843840,"num_objects":12815,"num_object_clones":0,"num_object_copies":38445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12815,"num_write_kb":820160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.788014"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":178,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195044+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:33:53.195044+0000","last_peered":"2023-10-09T06:33:53.195044+0000","last_clean":"2023-10-09T06:33:53.195044+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:33:53.195044+0000","last_undegraded":"2023-10-09T06:33:53.195044+0000","last_fullsized":"2023-10-09T06:33:53.195044+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179823+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:33:53.179823+0000","last_peered":"2023-10-09T06:33:53.179823+0000","last_clean":"2023-10-09T06:33:53.179823+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:33:53.179823+0000","last_undegraded":"2023-10-09T06:33:53.179823+0000","last_fullsized":"2023-10-09T06:33:53.179823+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188713+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:33:53.188713+0000","last_peered":"2023-10-09T06:33:53.188713+0000","last_clean":"2023-10-09T06:33:53.188713+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:33:53.188713+0000","last_undegraded":"2023-10-09T06:33:53.188713+0000","last_fullsized":"2023-10-09T06:33:53.188713+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190304+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:33:53.190304+0000","last_peered":"2023-10-09T06:33:53.190304+0000","last_clean":"2023-10-09T06:33:53.190304+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:33:53.190304+0000","last_undegraded":"2023-10-09T06:33:53.190304+0000","last_fullsized":"2023-10-09T06:33:53.190304+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:01.504835+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:34:01.504835+0000","last_peered":"2023-10-09T06:34:01.504835+0000","last_clean":"2023-10-09T06:34:01.504835+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:34:01.504835+0000","last_undegraded":"2023-10-09T06:34:01.504835+0000","last_fullsized":"2023-10-09T06:34:01.504835+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":37,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190244+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:33:53.190244+0000","last_peered":"2023-10-09T06:33:53.190244+0000","last_clean":"2023-10-09T06:33:53.190244+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:33:53.190244+0000","last_undegraded":"2023-10-09T06:33:53.190244+0000","last_fullsized":"2023-10-09T06:33:53.190244+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190334+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:33:53.190334+0000","last_peered":"2023-10-09T06:33:53.190334+0000","last_clean":"2023-10-09T06:33:53.190334+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:33:53.190334+0000","last_undegraded":"2023-10-09T06:33:53.190334+0000","last_fullsized":"2023-10-09T06:33:53.190334+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":204,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179435+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:33:53.179435+0000","last_peered":"2023-10-09T06:33:53.179435+0000","last_clean":"2023-10-09T06:33:53.179435+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:33:53.179435+0000","last_undegraded":"2023-10-09T06:33:53.179435+0000","last_fullsized":"2023-10-09T06:33:53.179435+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197591+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:33:53.197591+0000","last_peered":"2023-10-09T06:33:53.197591+0000","last_clean":"2023-10-09T06:33:53.197591+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:33:53.197591+0000","last_undegraded":"2023-10-09T06:33:53.197591+0000","last_fullsized":"2023-10-09T06:33:53.197591+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179205+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:33:53.179205+0000","last_peered":"2023-10-09T06:33:53.179205+0000","last_clean":"2023-10-09T06:33:53.179205+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:33:53.179205+0000","last_undegraded":"2023-10-09T06:33:53.179205+0000","last_fullsized":"2023-10-09T06:33:53.179205+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":95,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197512+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:33:53.197512+0000","last_peered":"2023-10-09T06:33:53.197512+0000","last_clean":"2023-10-09T06:33:53.197512+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:33:53.197512+0000","last_undegraded":"2023-10-09T06:33:53.197512+0000","last_fullsized":"2023-10-09T06:33:53.197512+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189308+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:33:53.189308+0000","last_peered":"2023-10-09T06:33:53.189308+0000","last_clean":"2023-10-09T06:33:53.189308+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:33:53.189308+0000","last_undegraded":"2023-10-09T06:33:53.189308+0000","last_fullsized":"2023-10-09T06:33:53.189308+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":245,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196341+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:33:53.196341+0000","last_peered":"2023-10-09T06:33:53.196341+0000","last_clean":"2023-10-09T06:33:53.196341+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:33:53.196341+0000","last_undegraded":"2023-10-09T06:33:53.196341+0000","last_fullsized":"2023-10-09T06:33:53.196341+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195973+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:33:53.195973+0000","last_peered":"2023-10-09T06:33:53.195973+0000","last_clean":"2023-10-09T06:33:53.195973+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:33:53.195973+0000","last_undegraded":"2023-10-09T06:33:53.195973+0000","last_fullsized":"2023-10-09T06:33:53.195973+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196675+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:33:53.196675+0000","last_peered":"2023-10-09T06:33:53.196675+0000","last_clean":"2023-10-09T06:33:53.196675+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:33:53.196675+0000","last_undegraded":"2023-10-09T06:33:53.196675+0000","last_fullsized":"2023-10-09T06:33:53.196675+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":191,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179490+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:33:53.179490+0000","last_peered":"2023-10-09T06:33:53.179490+0000","last_clean":"2023-10-09T06:33:53.179490+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:33:53.179490+0000","last_undegraded":"2023-10-09T06:33:53.179490+0000","last_fullsized":"2023-10-09T06:33:53.179490+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":171,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215283+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:34:08.215283+0000","last_peered":"2023-10-09T06:34:08.215283+0000","last_clean":"2023-10-09T06:34:08.215283+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:34:08.215283+0000","last_undegraded":"2023-10-09T06:34:08.215283+0000","last_fullsized":"2023-10-09T06:34:08.215283+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194987+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:33:53.194987+0000","last_peered":"2023-10-09T06:33:53.194987+0000","last_clean":"2023-10-09T06:33:53.194987+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:33:53.194987+0000","last_undegraded":"2023-10-09T06:33:53.194987+0000","last_fullsized":"2023-10-09T06:33:53.194987+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196318+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:33:53.196318+0000","last_peered":"2023-10-09T06:33:53.196318+0000","last_clean":"2023-10-09T06:33:53.196318+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:33:53.196318+0000","last_undegraded":"2023-10-09T06:33:53.196318+0000","last_fullsized":"2023-10-09T06:33:53.196318+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190261+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:33:53.190261+0000","last_peered":"2023-10-09T06:33:53.190261+0000","last_clean":"2023-10-09T06:33:53.190261+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:33:53.190261+0000","last_undegraded":"2023-10-09T06:33:53.190261+0000","last_fullsized":"2023-10-09T06:33:53.190261+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":47,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189348+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:33:53.189348+0000","last_peered":"2023-10-09T06:33:53.189348+0000","last_clean":"2023-10-09T06:33:53.189348+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:33:53.189348+0000","last_undegraded":"2023-10-09T06:33:53.189348+0000","last_fullsized":"2023-10-09T06:33:53.189348+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":204,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:01.504915+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:34:01.504915+0000","last_peered":"2023-10-09T06:34:01.504915+0000","last_clean":"2023-10-09T06:34:01.504915+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:34:01.504915+0000","last_undegraded":"2023-10-09T06:34:01.504915+0000","last_fullsized":"2023-10-09T06:34:01.504915+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"197'4180","reported_seq":4193,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.686348+0000","last_change":"2023-10-09T06:33:33.324346+0000","last_active":"2023-10-09T06:34:39.686348+0000","last_peered":"2023-10-09T06:34:39.686348+0000","last_clean":"2023-10-09T06:34:39.686348+0000","last_became_active":"2023-10-09T06:33:33.323046+0000","last_became_peered":"2023-10-09T06:33:33.323046+0000","last_unstale":"2023-10-09T06:34:39.686348+0000","last_undegraded":"2023-10-09T06:34:39.686348+0000","last_fullsized":"2023-10-09T06:34:39.686348+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273940480,"num_objects":4180,"num_object_clones":0,"num_object_copies":12540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4180,"num_write_kb":267520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":93,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195388+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:33:53.195388+0000","last_peered":"2023-10-09T06:33:53.195388+0000","last_clean":"2023-10-09T06:33:53.195388+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:33:53.195388+0000","last_undegraded":"2023-10-09T06:33:53.195388+0000","last_fullsized":"2023-10-09T06:33:53.195388+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":182,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195691+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:33:53.195691+0000","last_peered":"2023-10-09T06:33:53.195691+0000","last_clean":"2023-10-09T06:33:53.195691+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:33:53.195691+0000","last_undegraded":"2023-10-09T06:33:53.195691+0000","last_fullsized":"2023-10-09T06:33:53.195691+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"197'3838","reported_seq":3851,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:34.903288+0000","last_change":"2023-10-09T06:33:33.593835+0000","last_active":"2023-10-09T06:34:34.903288+0000","last_peered":"2023-10-09T06:34:34.903288+0000","last_clean":"2023-10-09T06:34:34.903288+0000","last_became_active":"2023-10-09T06:33:33.593363+0000","last_became_peered":"2023-10-09T06:33:33.593363+0000","last_unstale":"2023-10-09T06:34:34.903288+0000","last_undegraded":"2023-10-09T06:34:34.903288+0000","last_fullsized":"2023-10-09T06:34:34.903288+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":3838,"log_dups_size":0,"ondisk_log_size":3838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":245632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":93,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197341+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:33:53.197341+0000","last_peered":"2023-10-09T06:33:53.197341+0000","last_clean":"2023-10-09T06:33:53.197341+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:33:53.197341+0000","last_undegraded":"2023-10-09T06:33:53.197341+0000","last_fullsized":"2023-10-09T06:33:53.197341+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194805+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:33:53.194805+0000","last_peered":"2023-10-09T06:33:53.194805+0000","last_clean":"2023-10-09T06:33:53.194805+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:33:53.194805+0000","last_undegraded":"2023-10-09T06:33:53.194805+0000","last_fullsized":"2023-10-09T06:33:53.194805+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"197'4142","reported_seq":4156,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:38.598841+0000","last_change":"2023-10-09T06:33:33.587458+0000","last_active":"2023-10-09T06:34:38.598841+0000","last_peered":"2023-10-09T06:34:38.598841+0000","last_clean":"2023-10-09T06:34:38.598841+0000","last_became_active":"2023-10-09T06:33:33.586581+0000","last_became_peered":"2023-10-09T06:33:33.586581+0000","last_unstale":"2023-10-09T06:34:38.598841+0000","last_undegraded":"2023-10-09T06:34:38.598841+0000","last_fullsized":"2023-10-09T06:34:38.598841+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4142,"log_dups_size":0,"ondisk_log_size":4142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.486216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4142,"num_write_kb":265088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":119,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197152+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:33:53.197152+0000","last_peered":"2023-10-09T06:33:53.197152+0000","last_clean":"2023-10-09T06:33:53.197152+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:33:53.197152+0000","last_undegraded":"2023-10-09T06:33:53.197152+0000","last_fullsized":"2023-10-09T06:33:53.197152+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":37,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189419+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:33:53.189419+0000","last_peered":"2023-10-09T06:33:53.189419+0000","last_clean":"2023-10-09T06:33:53.189419+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:33:53.189419+0000","last_undegraded":"2023-10-09T06:33:53.189419+0000","last_fullsized":"2023-10-09T06:33:53.189419+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"197'4059","reported_seq":4071,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:37.430550+0000","last_change":"2023-10-09T06:33:33.319454+0000","last_active":"2023-10-09T06:34:37.430550+0000","last_peered":"2023-10-09T06:34:37.430550+0000","last_clean":"2023-10-09T06:34:37.430550+0000","last_became_active":"2023-10-09T06:33:33.318032+0000","last_became_peered":"2023-10-09T06:33:33.318032+0000","last_unstale":"2023-10-09T06:34:37.430550+0000","last_undegraded":"2023-10-09T06:34:37.430550+0000","last_fullsized":"2023-10-09T06:34:37.430550+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4059,"log_dups_size":0,"ondisk_log_size":4059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:46.906949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266010624,"num_objects":4059,"num_object_clones":0,"num_object_copies":12177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4059,"num_write_kb":259776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195345+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:33:53.195345+0000","last_peered":"2023-10-09T06:33:53.195345+0000","last_clean":"2023-10-09T06:33:53.195345+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:33:53.195345+0000","last_undegraded":"2023-10-09T06:33:53.195345+0000","last_fullsized":"2023-10-09T06:33:53.195345+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189796+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:33:53.189796+0000","last_peered":"2023-10-09T06:33:53.189796+0000","last_clean":"2023-10-09T06:33:53.189796+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:33:53.189796+0000","last_undegraded":"2023-10-09T06:33:53.189796+0000","last_fullsized":"2023-10-09T06:33:53.189796+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"197'4039","reported_seq":4053,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:36.539622+0000","last_change":"2023-10-09T06:33:33.320528+0000","last_active":"2023-10-09T06:34:36.539622+0000","last_peered":"2023-10-09T06:34:36.539622+0000","last_clean":"2023-10-09T06:34:36.539622+0000","last_became_active":"2023-10-09T06:33:33.320042+0000","last_became_peered":"2023-10-09T06:33:33.320042+0000","last_unstale":"2023-10-09T06:34:36.539622+0000","last_undegraded":"2023-10-09T06:34:36.539622+0000","last_fullsized":"2023-10-09T06:34:36.539622+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4039,"log_dups_size":0,"ondisk_log_size":4039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.099867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264699904,"num_objects":4039,"num_object_clones":0,"num_object_copies":12117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4039,"num_write_kb":258496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179796+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:33:53.179796+0000","last_peered":"2023-10-09T06:33:53.179796+0000","last_clean":"2023-10-09T06:33:53.179796+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:33:53.179796+0000","last_undegraded":"2023-10-09T06:33:53.179796+0000","last_fullsized":"2023-10-09T06:33:53.179796+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196072+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:33:53.196072+0000","last_peered":"2023-10-09T06:33:53.196072+0000","last_clean":"2023-10-09T06:33:53.196072+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:33:53.196072+0000","last_undegraded":"2023-10-09T06:33:53.196072+0000","last_fullsized":"2023-10-09T06:33:53.196072+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"197'4052","reported_seq":4066,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:37.459385+0000","last_change":"2023-10-09T06:33:33.321209+0000","last_active":"2023-10-09T06:34:37.459385+0000","last_peered":"2023-10-09T06:34:37.459385+0000","last_clean":"2023-10-09T06:34:37.459385+0000","last_became_active":"2023-10-09T06:33:33.320866+0000","last_became_peered":"2023-10-09T06:33:33.320866+0000","last_unstale":"2023-10-09T06:34:37.459385+0000","last_undegraded":"2023-10-09T06:34:37.459385+0000","last_fullsized":"2023-10-09T06:34:37.459385+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4052,"log_dups_size":0,"ondisk_log_size":4052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:34.914529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4052,"num_write_kb":259328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195254+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:33:53.195254+0000","last_peered":"2023-10-09T06:33:53.195254+0000","last_clean":"2023-10-09T06:33:53.195254+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:33:53.195254+0000","last_undegraded":"2023-10-09T06:33:53.195254+0000","last_fullsized":"2023-10-09T06:33:53.195254+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196465+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:33:53.196465+0000","last_peered":"2023-10-09T06:33:53.196465+0000","last_clean":"2023-10-09T06:33:53.196465+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:33:53.196465+0000","last_undegraded":"2023-10-09T06:33:53.196465+0000","last_fullsized":"2023-10-09T06:33:53.196465+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"197'4258","reported_seq":4271,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.629994+0000","last_change":"2023-10-09T06:33:33.244614+0000","last_active":"2023-10-09T06:34:39.629994+0000","last_peered":"2023-10-09T06:34:39.629994+0000","last_clean":"2023-10-09T06:34:39.629994+0000","last_became_active":"2023-10-09T06:33:33.243719+0000","last_became_peered":"2023-10-09T06:33:33.243719+0000","last_unstale":"2023-10-09T06:34:39.629994+0000","last_undegraded":"2023-10-09T06:34:39.629994+0000","last_fullsized":"2023-10-09T06:34:39.629994+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4258,"log_dups_size":0,"ondisk_log_size":4258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279052288,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4258,"num_write_kb":272512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194482+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:33:53.194482+0000","last_peered":"2023-10-09T06:33:53.194482+0000","last_clean":"2023-10-09T06:33:53.194482+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:33:53.194482+0000","last_undegraded":"2023-10-09T06:33:53.194482+0000","last_fullsized":"2023-10-09T06:33:53.194482+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196345+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:33:53.196345+0000","last_peered":"2023-10-09T06:33:53.196345+0000","last_clean":"2023-10-09T06:33:53.196345+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:33:53.196345+0000","last_undegraded":"2023-10-09T06:33:53.196345+0000","last_fullsized":"2023-10-09T06:33:53.196345+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"197'4029","reported_seq":4043,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:37.443683+0000","last_change":"2023-10-09T06:33:33.319285+0000","last_active":"2023-10-09T06:34:37.443683+0000","last_peered":"2023-10-09T06:34:37.443683+0000","last_clean":"2023-10-09T06:34:37.443683+0000","last_became_active":"2023-10-09T06:33:33.318947+0000","last_became_peered":"2023-10-09T06:33:33.318947+0000","last_unstale":"2023-10-09T06:34:37.443683+0000","last_undegraded":"2023-10-09T06:34:37.443683+0000","last_fullsized":"2023-10-09T06:34:37.443683+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4029,"log_dups_size":0,"ondisk_log_size":4029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:22.467861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264044544,"num_objects":4029,"num_object_clones":0,"num_object_copies":12087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4029,"num_write_kb":257856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":193,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.219256+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:34:08.219256+0000","last_peered":"2023-10-09T06:34:08.219256+0000","last_clean":"2023-10-09T06:34:08.219256+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:34:08.219256+0000","last_undegraded":"2023-10-09T06:34:08.219256+0000","last_fullsized":"2023-10-09T06:34:08.219256+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:57.404378+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:33:57.404378+0000","last_peered":"2023-10-09T06:33:57.404378+0000","last_clean":"2023-10-09T06:33:57.404378+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:33:57.404378+0000","last_undegraded":"2023-10-09T06:33:57.404378+0000","last_fullsized":"2023-10-09T06:33:57.404378+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"197'4150","reported_seq":4164,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:38.619604+0000","last_change":"2023-10-09T06:33:33.317867+0000","last_active":"2023-10-09T06:34:38.619604+0000","last_peered":"2023-10-09T06:34:38.619604+0000","last_clean":"2023-10-09T06:34:38.619604+0000","last_became_active":"2023-10-09T06:33:33.317504+0000","last_became_peered":"2023-10-09T06:33:33.317504+0000","last_unstale":"2023-10-09T06:34:38.619604+0000","last_undegraded":"2023-10-09T06:34:38.619604+0000","last_fullsized":"2023-10-09T06:34:38.619604+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4150,"log_dups_size":0,"ondisk_log_size":4150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:56.906981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271974400,"num_objects":4150,"num_object_clones":0,"num_object_copies":12450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4150,"num_write_kb":265600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1051,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213345+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:34:08.213345+0000","last_peered":"2023-10-09T06:34:08.213345+0000","last_clean":"2023-10-09T06:34:08.213345+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:34:08.213345+0000","last_undegraded":"2023-10-09T06:34:08.213345+0000","last_fullsized":"2023-10-09T06:34:08.213345+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197546+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:33:53.197546+0000","last_peered":"2023-10-09T06:33:53.197546+0000","last_clean":"2023-10-09T06:33:53.197546+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:33:53.197546+0000","last_undegraded":"2023-10-09T06:33:53.197546+0000","last_fullsized":"2023-10-09T06:33:53.197546+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213544+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:34:08.213544+0000","last_peered":"2023-10-09T06:34:08.213544+0000","last_clean":"2023-10-09T06:34:08.213544+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:34:08.213544+0000","last_undegraded":"2023-10-09T06:34:08.213544+0000","last_fullsized":"2023-10-09T06:34:08.213544+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"197'4141","reported_seq":4154,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.692191+0000","last_change":"2023-10-09T06:33:33.593915+0000","last_active":"2023-10-09T06:34:39.692191+0000","last_peered":"2023-10-09T06:34:39.692191+0000","last_clean":"2023-10-09T06:34:39.692191+0000","last_became_active":"2023-10-09T06:33:33.593144+0000","last_became_peered":"2023-10-09T06:33:33.593144+0000","last_unstale":"2023-10-09T06:34:39.692191+0000","last_undegraded":"2023-10-09T06:34:39.692191+0000","last_fullsized":"2023-10-09T06:34:39.692191+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4141,"log_dups_size":0,"ondisk_log_size":4141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271384576,"num_objects":4141,"num_object_clones":0,"num_object_copies":12423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4141,"num_write_kb":265024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":835,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195371+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:33:53.195371+0000","last_peered":"2023-10-09T06:33:53.195371+0000","last_clean":"2023-10-09T06:33:53.195371+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:33:53.195371+0000","last_undegraded":"2023-10-09T06:33:53.195371+0000","last_fullsized":"2023-10-09T06:33:53.195371+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":180,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195951+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:33:53.195951+0000","last_peered":"2023-10-09T06:33:53.195951+0000","last_clean":"2023-10-09T06:33:53.195951+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:33:53.195951+0000","last_undegraded":"2023-10-09T06:33:53.195951+0000","last_fullsized":"2023-10-09T06:33:53.195951+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213188+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:34:08.213188+0000","last_peered":"2023-10-09T06:34:08.213188+0000","last_clean":"2023-10-09T06:34:08.213188+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:34:08.213188+0000","last_undegraded":"2023-10-09T06:34:08.213188+0000","last_fullsized":"2023-10-09T06:34:08.213188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"197'3934","reported_seq":3948,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:36.555410+0000","last_change":"2023-10-09T06:33:33.319622+0000","last_active":"2023-10-09T06:34:36.555410+0000","last_peered":"2023-10-09T06:34:36.555410+0000","last_clean":"2023-10-09T06:34:36.555410+0000","last_became_active":"2023-10-09T06:33:33.319140+0000","last_became_peered":"2023-10-09T06:33:33.319140+0000","last_unstale":"2023-10-09T06:34:36.555410+0000","last_undegraded":"2023-10-09T06:34:36.555410+0000","last_fullsized":"2023-10-09T06:34:36.555410+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":3934,"log_dups_size":0,"ondisk_log_size":3934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:00.867299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257818624,"num_objects":3934,"num_object_clones":0,"num_object_copies":11802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3934,"num_write_kb":251776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":561,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189207+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:33:53.189207+0000","last_peered":"2023-10-09T06:33:53.189207+0000","last_clean":"2023-10-09T06:33:53.189207+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:33:53.189207+0000","last_undegraded":"2023-10-09T06:33:53.189207+0000","last_fullsized":"2023-10-09T06:33:53.189207+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.188520+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:33:53.188520+0000","last_peered":"2023-10-09T06:33:53.188520+0000","last_clean":"2023-10-09T06:33:53.188520+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:33:53.188520+0000","last_undegraded":"2023-10-09T06:33:53.188520+0000","last_fullsized":"2023-10-09T06:33:53.188520+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190699+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:33:53.190699+0000","last_peered":"2023-10-09T06:33:53.190699+0000","last_clean":"2023-10-09T06:33:53.190699+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:33:53.190699+0000","last_undegraded":"2023-10-09T06:33:53.190699+0000","last_fullsized":"2023-10-09T06:33:53.190699+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"197'4234","reported_seq":4247,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.670234+0000","last_change":"2023-10-09T06:33:33.244731+0000","last_active":"2023-10-09T06:34:39.670234+0000","last_peered":"2023-10-09T06:34:39.670234+0000","last_clean":"2023-10-09T06:34:39.670234+0000","last_became_active":"2023-10-09T06:33:33.243726+0000","last_became_peered":"2023-10-09T06:33:33.243726+0000","last_unstale":"2023-10-09T06:34:39.670234+0000","last_undegraded":"2023-10-09T06:34:39.670234+0000","last_fullsized":"2023-10-09T06:34:39.670234+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4234,"log_dups_size":0,"ondisk_log_size":4234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4234,"num_write_kb":270976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":545,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194774+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:33:53.194774+0000","last_peered":"2023-10-09T06:33:53.194774+0000","last_clean":"2023-10-09T06:33:53.194774+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:33:53.194774+0000","last_undegraded":"2023-10-09T06:33:53.194774+0000","last_fullsized":"2023-10-09T06:33:53.194774+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":439,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195883+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:33:53.195883+0000","last_peered":"2023-10-09T06:33:53.195883+0000","last_clean":"2023-10-09T06:33:53.195883+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:33:53.195883+0000","last_undegraded":"2023-10-09T06:33:53.195883+0000","last_fullsized":"2023-10-09T06:33:53.195883+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":183,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179196+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:33:53.179196+0000","last_peered":"2023-10-09T06:33:53.179196+0000","last_clean":"2023-10-09T06:33:53.179196+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:33:53.179196+0000","last_undegraded":"2023-10-09T06:33:53.179196+0000","last_fullsized":"2023-10-09T06:33:53.179196+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196594+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:33:53.196594+0000","last_peered":"2023-10-09T06:33:53.196594+0000","last_clean":"2023-10-09T06:33:53.196594+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:33:53.196594+0000","last_undegraded":"2023-10-09T06:33:53.196594+0000","last_fullsized":"2023-10-09T06:33:53.196594+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"197'4091","reported_seq":4104,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:37.449765+0000","last_change":"2023-10-09T06:33:33.319537+0000","last_active":"2023-10-09T06:34:37.449765+0000","last_peered":"2023-10-09T06:34:37.449765+0000","last_clean":"2023-10-09T06:34:37.449765+0000","last_became_active":"2023-10-09T06:33:33.319150+0000","last_became_peered":"2023-10-09T06:33:33.319150+0000","last_unstale":"2023-10-09T06:34:37.449765+0000","last_undegraded":"2023-10-09T06:34:37.449765+0000","last_fullsized":"2023-10-09T06:34:37.449765+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4091,"log_dups_size":0,"ondisk_log_size":4091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:18.214528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268107776,"num_objects":4091,"num_object_clones":0,"num_object_copies":12273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4091,"num_write_kb":261824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":118,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195494+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:33:53.195494+0000","last_peered":"2023-10-09T06:33:53.195494+0000","last_clean":"2023-10-09T06:33:53.195494+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:33:53.195494+0000","last_undegraded":"2023-10-09T06:33:53.195494+0000","last_fullsized":"2023-10-09T06:33:53.195494+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713035+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:34:03.713035+0000","last_peered":"2023-10-09T06:34:03.713035+0000","last_clean":"2023-10-09T06:34:03.713035+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:34:03.713035+0000","last_undegraded":"2023-10-09T06:34:03.713035+0000","last_fullsized":"2023-10-09T06:34:03.713035+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190411+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:33:53.190411+0000","last_peered":"2023-10-09T06:33:53.190411+0000","last_clean":"2023-10-09T06:33:53.190411+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:33:53.190411+0000","last_undegraded":"2023-10-09T06:33:53.190411+0000","last_fullsized":"2023-10-09T06:33:53.190411+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"197'3946","reported_seq":3960,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:36.564707+0000","last_change":"2023-10-09T06:33:33.320380+0000","last_active":"2023-10-09T06:34:36.564707+0000","last_peered":"2023-10-09T06:34:36.564707+0000","last_clean":"2023-10-09T06:34:36.564707+0000","last_became_active":"2023-10-09T06:33:33.319960+0000","last_became_peered":"2023-10-09T06:33:33.319960+0000","last_unstale":"2023-10-09T06:34:36.564707+0000","last_undegraded":"2023-10-09T06:34:36.564707+0000","last_fullsized":"2023-10-09T06:34:36.564707+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":3946,"log_dups_size":0,"ondisk_log_size":3946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:25.704605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3946,"num_write_kb":252544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":107,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197259+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:33:53.197259+0000","last_peered":"2023-10-09T06:33:53.197259+0000","last_clean":"2023-10-09T06:33:53.197259+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:33:53.197259+0000","last_undegraded":"2023-10-09T06:33:53.197259+0000","last_fullsized":"2023-10-09T06:33:53.197259+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213268+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:34:08.213268+0000","last_peered":"2023-10-09T06:34:08.213268+0000","last_clean":"2023-10-09T06:34:08.213268+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:34:08.213268+0000","last_undegraded":"2023-10-09T06:34:08.213268+0000","last_fullsized":"2023-10-09T06:34:08.213268+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":166,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713069+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:34:03.713069+0000","last_peered":"2023-10-09T06:34:03.713069+0000","last_clean":"2023-10-09T06:34:03.713069+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:34:03.713069+0000","last_undegraded":"2023-10-09T06:34:03.713069+0000","last_fullsized":"2023-10-09T06:34:03.713069+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"197'4278","reported_seq":4292,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:38.582431+0000","last_change":"2023-10-09T06:33:33.586554+0000","last_active":"2023-10-09T06:34:38.582431+0000","last_peered":"2023-10-09T06:34:38.582431+0000","last_clean":"2023-10-09T06:34:38.582431+0000","last_became_active":"2023-10-09T06:33:33.586066+0000","last_became_peered":"2023-10-09T06:33:33.586066+0000","last_unstale":"2023-10-09T06:34:38.582431+0000","last_undegraded":"2023-10-09T06:34:38.582431+0000","last_fullsized":"2023-10-09T06:34:38.582431+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4278,"log_dups_size":0,"ondisk_log_size":4278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.973752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280363008,"num_objects":4278,"num_object_clones":0,"num_object_copies":12834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4278,"num_write_kb":273792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":199,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.228427+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:34:08.228427+0000","last_peered":"2023-10-09T06:34:08.228427+0000","last_clean":"2023-10-09T06:34:08.228427+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:34:08.228427+0000","last_undegraded":"2023-10-09T06:34:08.228427+0000","last_fullsized":"2023-10-09T06:34:08.228427+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":580,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190686+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:33:53.190686+0000","last_peered":"2023-10-09T06:33:53.190686+0000","last_clean":"2023-10-09T06:33:53.190686+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:33:53.190686+0000","last_undegraded":"2023-10-09T06:33:53.190686+0000","last_fullsized":"2023-10-09T06:33:53.190686+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179154+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:33:53.179154+0000","last_peered":"2023-10-09T06:33:53.179154+0000","last_clean":"2023-10-09T06:33:53.179154+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:33:53.179154+0000","last_undegraded":"2023-10-09T06:33:53.179154+0000","last_fullsized":"2023-10-09T06:33:53.179154+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"197'4135","reported_seq":4148,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.681734+0000","last_change":"2023-10-09T06:33:33.594495+0000","last_active":"2023-10-09T06:34:39.681734+0000","last_peered":"2023-10-09T06:34:39.681734+0000","last_clean":"2023-10-09T06:34:39.681734+0000","last_became_active":"2023-10-09T06:33:33.593206+0000","last_became_peered":"2023-10-09T06:33:33.593206+0000","last_unstale":"2023-10-09T06:34:39.681734+0000","last_undegraded":"2023-10-09T06:34:39.681734+0000","last_fullsized":"2023-10-09T06:34:39.681734+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":4135,"log_dups_size":0,"ondisk_log_size":4135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4135,"num_write_kb":264640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":717,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190257+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:33:53.190257+0000","last_peered":"2023-10-09T06:33:53.190257+0000","last_clean":"2023-10-09T06:33:53.190257+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:33:53.190257+0000","last_undegraded":"2023-10-09T06:33:53.190257+0000","last_fullsized":"2023-10-09T06:33:53.190257+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.190347+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:33:53.190347+0000","last_peered":"2023-10-09T06:33:53.190347+0000","last_clean":"2023-10-09T06:33:53.190347+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:33:53.190347+0000","last_undegraded":"2023-10-09T06:33:53.190347+0000","last_fullsized":"2023-10-09T06:33:53.190347+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.199037+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:33:53.199037+0000","last_peered":"2023-10-09T06:33:53.199037+0000","last_clean":"2023-10-09T06:33:53.199037+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:33:53.199037+0000","last_undegraded":"2023-10-09T06:33:53.199037+0000","last_fullsized":"2023-10-09T06:33:53.199037+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.189956+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:33:53.189956+0000","last_peered":"2023-10-09T06:33:53.189956+0000","last_clean":"2023-10-09T06:33:53.189956+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:33:53.189956+0000","last_undegraded":"2023-10-09T06:33:53.189956+0000","last_fullsized":"2023-10-09T06:33:53.189956+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.222382+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:34:08.222382+0000","last_peered":"2023-10-09T06:34:08.222382+0000","last_clean":"2023-10-09T06:34:08.222382+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:34:08.222382+0000","last_undegraded":"2023-10-09T06:34:08.222382+0000","last_fullsized":"2023-10-09T06:34:08.222382+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":164,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195587+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:33:53.195587+0000","last_peered":"2023-10-09T06:33:53.195587+0000","last_clean":"2023-10-09T06:33:53.195587+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:33:53.195587+0000","last_undegraded":"2023-10-09T06:33:53.195587+0000","last_fullsized":"2023-10-09T06:33:53.195587+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.196153+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:33:53.196153+0000","last_peered":"2023-10-09T06:33:53.196153+0000","last_clean":"2023-10-09T06:33:53.196153+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:33:53.196153+0000","last_undegraded":"2023-10-09T06:33:53.196153+0000","last_fullsized":"2023-10-09T06:33:53.196153+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":175,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.197544+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:33:53.197544+0000","last_peered":"2023-10-09T06:33:53.197544+0000","last_clean":"2023-10-09T06:33:53.197544+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:33:53.197544+0000","last_undegraded":"2023-10-09T06:33:53.197544+0000","last_fullsized":"2023-10-09T06:33:53.197544+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.195444+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:33:53.195444+0000","last_peered":"2023-10-09T06:33:53.195444+0000","last_clean":"2023-10-09T06:33:53.195444+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:33:53.195444+0000","last_undegraded":"2023-10-09T06:33:53.195444+0000","last_fullsized":"2023-10-09T06:33:53.195444+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":198,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.179885+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:33:53.179885+0000","last_peered":"2023-10-09T06:33:53.179885+0000","last_clean":"2023-10-09T06:33:53.179885+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:33:53.179885+0000","last_undegraded":"2023-10-09T06:33:53.179885+0000","last_fullsized":"2023-10-09T06:33:53.179885+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":192,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:06.507183+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:34:06.507183+0000","last_peered":"2023-10-09T06:34:06.507183+0000","last_clean":"2023-10-09T06:34:06.507183+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:34:06.507183+0000","last_undegraded":"2023-10-09T06:34:06.507183+0000","last_fullsized":"2023-10-09T06:34:06.507183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":158,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:34:03.713130+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:34:03.713130+0000","last_peered":"2023-10-09T06:34:03.713130+0000","last_clean":"2023-10-09T06:34:03.713130+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:34:03.713130+0000","last_undegraded":"2023-10-09T06:34:03.713130+0000","last_fullsized":"2023-10-09T06:34:03.713130+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-10-09T06:33:53.194395+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:33:53.194395+0000","last_peered":"2023-10-09T06:33:53.194395+0000","last_clean":"2023-10-09T06:33:53.194395+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:33:53.194395+0000","last_undegraded":"2023-10-09T06:33:53.194395+0000","last_fullsized":"2023-10-09T06:33:53.194395+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4293001216,"num_objects":65506,"num_object_clones":0,"num_object_copies":196518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":65506,"num_write_kb":4192384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12854296576,"data_stored":12854296576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65506,"ondisk_log_size":65506,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430640,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1629860,"kb_used_data":1332636,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92115292,"statfs":{"total":95995035648,"available":94326059008,"internally_reserved":0,"allocated":1364619264,"data_stored":1364062141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":3.8769999999999998,"15min":2.9300000000000002},"min":{"1min":0.50700000000000001,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":1.367,"5min":178.738,"15min":178.738},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":3.8719999999999999,"15min":2.8690000000000002},"min":{"1min":0.55000000000000004,"5min":0.252,"15min":0.252},"max":{"1min":1.1950000000000001,"5min":177.75,"15min":177.75},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":7.8849999999999998,"5min":2.552,"15min":2.4940000000000002},"min":{"1min":0.38400000000000001,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":148.88499999999999,"5min":148.88499999999999,"15min":148.88499999999999},"last":4.0999999999999996},{"interface":"front","average":{"1min":7.7610000000000001,"5min":2.6909999999999998,"15min":2.3809999999999998},"min":{"1min":0.38200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":148.059,"5min":148.059,"15min":148.059},"last":0.378}]},{"osd":2,"last update":"Mon Oct 9 06:34:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.2849999999999999,"15min":2.4100000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29399999999999998},"max":{"1min":2.8180000000000001,"5min":15.585000000000001,"15min":104.992},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.3160000000000001,"15min":2.2930000000000001},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.456,"5min":7.0629999999999997,"15min":104.691},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.7890000000000001,"15min":3.573},"min":{"1min":0.42499999999999999,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":3.323,"5min":90.328000000000003,"15min":183.971},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.5590000000000002,"15min":3.3599999999999999},"min":{"1min":0.48099999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.5339999999999998,"5min":90.295000000000002,"15min":183.095},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.018,"15min":1.2370000000000001},"min":{"1min":0.57699999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.397,"5min":6.6239999999999997,"15min":11.029999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.002,"5min":1.298,"15min":1.5980000000000001},"min":{"1min":0.54000000000000004,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.341,"5min":15.504,"15min":15.504},"last":2.8439999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0449999999999999,"15min":1.3999999999999999},"min":{"1min":0.626,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.4450000000000001,"5min":7.1550000000000002,"15min":11.996},"last":1.9019999999999999},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.012,"15min":1.3939999999999999},"min":{"1min":0.64200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.631,"5min":5.4909999999999997,"15min":13.634},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.4259999999999999,"15min":1.3260000000000001},"min":{"1min":0.64900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":1.6879999999999999,"5min":15.521000000000001,"15min":15.521000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.115,"5min":1.347,"15min":1.5049999999999999},"min":{"1min":0.496,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":1.5109999999999999,"5min":15.449999999999999,"15min":15.449999999999999},"last":3.1269999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626871,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1544084,"kb_used_data":1242764,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92201068,"statfs":{"total":95995035648,"available":94413893632,"internally_reserved":0,"allocated":1272590336,"data_stored":1272048840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.319,"15min":1.379},"min":{"1min":0.64800000000000002,"5min":0.27200000000000002,"15min":0.26300000000000001},"max":{"1min":1.2969999999999999,"5min":9.4649999999999999,"15min":32.289999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.2649999999999999,"15min":1.48},"min":{"1min":0.31900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.2270000000000001,"5min":6.4240000000000004,"15min":31.454999999999998},"last":0.90400000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":13.691000000000001,"5min":3.875,"15min":2.673},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":316.88099999999997,"5min":316.88099999999997,"15min":316.88099999999997},"last":0.98699999999999999},{"interface":"front","average":{"1min":13.871,"5min":4.0250000000000004,"15min":2.7189999999999999},"min":{"1min":0.39900000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":316.84399999999999,"5min":316.84399999999999,"15min":316.84399999999999},"last":0.73399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":2.4740000000000002,"15min":2.3900000000000001},"min":{"1min":0.56000000000000005,"5min":0.35699999999999998,"15min":0.32000000000000001},"max":{"1min":2.2839999999999998,"5min":53.616,"15min":95.540000000000006},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":2.6120000000000001,"15min":2.3650000000000002},"min":{"1min":0.61099999999999999,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":1.454,"5min":53.665999999999997,"15min":96.352000000000004},"last":1.085}]},{"osd":3,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.23,"15min":2.3599999999999999},"min":{"1min":0.77300000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":2.468,"5min":27.491,"15min":115.09399999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.8380000000000001,"15min":2.0089999999999999},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":1.3009999999999999,"5min":26.815000000000001,"15min":112.886},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.98399999999999999,"15min":1.2569999999999999},"min":{"1min":0.39200000000000002,"5min":0.32500000000000001,"15min":0.26000000000000001},"max":{"1min":2.6059999999999999,"5min":6.3799999999999999,"15min":7.3090000000000002},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0529999999999999,"15min":1.419},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":4.0890000000000004,"5min":5.1609999999999996,"15min":10.300000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":0.98399999999999999,"15min":1.4550000000000001},"min":{"1min":0.42799999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":4.9619999999999997,"5min":6.4850000000000003,"15min":11.754},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.1699999999999999,"15min":1.589},"min":{"1min":0.41899999999999998,"5min":0.38200000000000001,"15min":0.308},"max":{"1min":6.423,"5min":6.423,"15min":12.295999999999999},"last":0.92200000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.083,"15min":1.502},"min":{"1min":0.81299999999999994,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":1.651,"5min":6.5110000000000001,"15min":11.076000000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.27,"5min":1.127,"15min":1.4330000000000001},"min":{"1min":0.94999999999999996,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":2.52,"5min":9.4179999999999993,"15min":11.102},"last":1.1719999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958781,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2155924,"kb_used_data":1793164,"kb_used_omap":7,"kb_used_meta":362744,"kb_avail":91589228,"statfs":{"total":95995035648,"available":93787369472,"internally_reserved":0,"allocated":1836199936,"data_stored":1835367896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":371450256},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":4.0869999999999997,"15min":2.5680000000000001},"min":{"1min":0.33600000000000002,"5min":0.187,"15min":0.187},"max":{"1min":6.4960000000000004,"5min":132.886,"15min":132.886},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.321,"5min":4.0970000000000004,"15min":2.7650000000000001},"min":{"1min":0.33800000000000002,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":9.7409999999999997,"5min":133.315,"15min":133.315},"last":1.897}]},{"osd":2,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.6440000000000001,"15min":1.8740000000000001},"min":{"1min":0.35499999999999998,"5min":0.214,"15min":0.19800000000000001},"max":{"1min":1.569,"5min":67.206000000000003,"15min":67.206000000000003},"last":4.2329999999999997},{"interface":"front","average":{"1min":0.71899999999999997,"5min":2.923,"15min":1.948},"min":{"1min":0.309,"5min":0.224,"15min":0.224},"max":{"1min":1.099,"5min":67.132000000000005,"15min":67.132000000000005},"last":2.9489999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":5.0490000000000004,"15min":4.0359999999999996},"min":{"1min":0.44700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":4.4290000000000003,"5min":132.792,"15min":132.792},"last":2.5619999999999998},{"interface":"front","average":{"1min":0.89400000000000002,"5min":4.9589999999999996,"15min":3.9980000000000002},"min":{"1min":0.40300000000000002,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":3.1589999999999998,"5min":132.53299999999999,"15min":132.53299999999999},"last":0.44700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.5800000000000001,"15min":1.5149999999999999},"min":{"1min":0.66200000000000003,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":4.3360000000000003,"5min":26.068999999999999,"15min":26.068999999999999},"last":2.7410000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.391,"15min":1.5509999999999999},"min":{"1min":0.39100000000000001,"5min":0.311,"15min":0.31},"max":{"1min":2.7000000000000002,"5min":5.6689999999999996,"15min":12.907999999999999},"last":0.52800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.6299999999999999,"15min":1.6180000000000001},"min":{"1min":0.47199999999999998,"5min":0.35899999999999999,"15min":0.32500000000000001},"max":{"1min":10.218,"5min":10.586,"15min":14.170999999999999},"last":4.8819999999999997},{"interface":"front","average":{"1min":1.462,"5min":1.5600000000000001,"15min":1.6559999999999999},"min":{"1min":0.65600000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.0279999999999996,"5min":26.157,"15min":26.157},"last":2.7869999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.6399999999999999,"15min":1.538},"min":{"1min":0.502,"5min":0.38600000000000001,"15min":0.36699999999999999},"max":{"1min":10.19,"5min":10.625,"15min":14.212999999999999},"last":4.9189999999999996},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.663,"15min":1.6559999999999999},"min":{"1min":0.64400000000000002,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":4.3799999999999999,"5min":26.132000000000001,"15min":26.132000000000001},"last":4.2770000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.9059999999999999,"15min":1.988},"min":{"1min":0.63900000000000001,"5min":0.41099999999999998,"15min":0.35399999999999998},"max":{"1min":4.3620000000000001,"5min":26.109000000000002,"15min":30.774999999999999},"last":2.8140000000000001},{"interface":"front","average":{"1min":1.111,"5min":1.486,"15min":1.6799999999999999},"min":{"1min":0.439,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.7229999999999999,"5min":5.8289999999999997,"15min":30.815000000000001},"last":0.57099999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122307,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2087032,"kb_used_data":1606000,"kb_used_omap":8,"kb_used_meta":481015,"kb_avail":91658120,"statfs":{"total":95995035648,"available":93857914880,"internally_reserved":0,"allocated":1644544000,"data_stored":1643736851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":492559616},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":2.7629999999999999,"15min":2.5880000000000001},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4139999999999997,"5min":13.598000000000001,"15min":138.31700000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.95599999999999996,"5min":2.2400000000000002,"15min":2.355},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20200000000000001},"max":{"1min":3.8879999999999999,"5min":12.101000000000001,"15min":144.37},"last":2.1560000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":2.4929999999999999,"15min":1.9419999999999999},"min":{"1min":0.34699999999999998,"5min":0.185,"15min":0.185},"max":{"1min":0.68500000000000005,"5min":13.425000000000001,"15min":28.745000000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":2.0499999999999998,"15min":1.909},"min":{"1min":0.34999999999999998,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":1.1359999999999999,"5min":16.129000000000001,"15min":28.146999999999998},"last":3.0169999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.5129999999999999,"15min":2.3639999999999999},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.20899999999999999},"max":{"1min":6.4000000000000004,"5min":14.628,"15min":59.643000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.119,"5min":2.3929999999999998,"15min":2.2149999999999999},"min":{"1min":0.43099999999999999,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.956,"5min":12.791,"15min":59.762999999999998},"last":2.1989999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.611,"15min":1.5509999999999999},"min":{"1min":0.38500000000000001,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":1.2849999999999999,"5min":8.9920000000000009,"15min":9.0890000000000004},"last":0.434},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.966,"15min":1.8360000000000001},"min":{"1min":0.54400000000000004,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.9710000000000001,"5min":9.2319999999999993,"15min":28.629000000000001},"last":1.51}]},{"osd":5,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":2.0739999999999998,"15min":2.0630000000000002},"min":{"1min":0.70299999999999996,"5min":0.36199999999999999,"15min":0.35399999999999998},"max":{"1min":3.0059999999999998,"5min":12.356999999999999,"15min":28.574000000000002},"last":3.1389999999999998},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.873,"15min":1.8089999999999999},"min":{"1min":0.54000000000000004,"5min":0.36299999999999999,"15min":0.32600000000000001},"max":{"1min":1.613,"5min":9.8209999999999997,"15min":28.061},"last":2.2290000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.8360000000000001,"15min":1.738},"min":{"1min":0.59499999999999997,"5min":0.35299999999999998,"15min":0.34599999999999997},"max":{"1min":1.548,"5min":16.181999999999999,"15min":27.966999999999999},"last":2.258},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.6120000000000001,"15min":1.4710000000000001},"min":{"1min":0.58399999999999996,"5min":0.39300000000000002,"15min":0.33700000000000002},"max":{"1min":1.3300000000000001,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":0.40799999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8029999999999999,"15min":1.833},"min":{"1min":0.63200000000000001,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":1.6599999999999999,"5min":7.9169999999999998,"15min":28.109999999999999},"last":2.2850000000000001},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.8380000000000001,"15min":1.863},"min":{"1min":0.41799999999999998,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":1.5209999999999999,"5min":8.9190000000000005,"15min":28.074999999999999},"last":0.628}]}]},{"osd":2,"up_from":98,"seq":420906795255,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1941176,"kb_used_data":1502576,"kb_used_omap":11,"kb_used_meta":438580,"kb_avail":91803976,"statfs":{"total":95995035648,"available":94007271424,"internally_reserved":0,"allocated":1538637824,"data_stored":1537827967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":449106878},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.0790000000000002,"15min":2.2749999999999999},"min":{"1min":0.35199999999999998,"5min":0.189,"15min":0.17899999999999999},"max":{"1min":0.85899999999999999,"5min":117.2,"15min":117.2},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":2.6869999999999998,"15min":2.351},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":0.76300000000000001,"5min":121.521,"15min":121.521},"last":2.6989999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4569999999999999,"5min":2.133,"15min":1.8109999999999999},"min":{"1min":0.41899999999999998,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":60.155000000000001,"5min":60.155000000000001,"15min":60.155000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":3.5640000000000001,"5min":1.982,"15min":2.0219999999999998},"min":{"1min":0.35799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":62.582999999999998,"5min":62.582999999999998,"15min":62.582999999999998},"last":7.9989999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":3.1800000000000002,"15min":3.3250000000000002},"min":{"1min":0.253,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":2.1110000000000002,"5min":118.431,"15min":201.33600000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.70999999999999996,"5min":2.7799999999999998,"15min":3.2919999999999998},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":2.113,"5min":118.64100000000001,"15min":199.95099999999999},"last":0.56000000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.359,"15min":1.9850000000000001},"min":{"1min":0.624,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.093,"5min":7.5759999999999996,"15min":19.184000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.872,"5min":1.2070000000000001,"15min":1.7889999999999999},"min":{"1min":0.628,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":1.3360000000000001,"5min":6.3049999999999997,"15min":29.408000000000001},"last":0.51200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.335,"15min":2.3370000000000002},"min":{"1min":0.59399999999999997,"5min":0.35299999999999998,"15min":0.31},"max":{"1min":1.202,"5min":86.423000000000002,"15min":86.423000000000002},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.93300000000000005,"5min":2.2400000000000002,"15min":2.0179999999999998},"min":{"1min":0.58599999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.2829999999999999,"5min":86.382999999999996,"15min":86.382999999999996},"last":1.603}]},{"osd":6,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.4910000000000001,"15min":1.6419999999999999},"min":{"1min":0.70699999999999996,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":1.377,"5min":8.2959999999999994,"15min":29.504000000000001},"last":1.716},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.4570000000000001,"15min":1.647},"min":{"1min":0.65200000000000002,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":1.1890000000000001,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.595,"15min":1.772},"min":{"1min":0.69599999999999995,"5min":0.41599999999999998,"15min":0.308},"max":{"1min":1.2849999999999999,"5min":9.2050000000000001,"15min":9.2050000000000001},"last":0.747},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5600000000000001,"15min":2.0059999999999998},"min":{"1min":0.71399999999999997,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":1.2190000000000001,"5min":7.6920000000000002,"15min":12.051},"last":0.82499999999999996}]}]},{"osd":3,"up_from":103,"seq":442381631729,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1993088,"kb_used_data":1518328,"kb_used_omap":3,"kb_used_meta":474748,"kb_avail":91752064,"statfs":{"total":95995035648,"available":93954113536,"internally_reserved":0,"allocated":1554767872,"data_stored":1553950715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":486142497},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.3769999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17399999999999999},"max":{"1min":3.23,"5min":15.285,"15min":147.988},"last":1.669},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.2050000000000001,"15min":2.4329999999999998},"min":{"1min":0.19500000000000001,"5min":0.19,"15min":0.18099999999999999},"max":{"1min":4.9589999999999996,"5min":23.646000000000001,"15min":147.94200000000001},"last":1.3169999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":2.5409999999999999,"15min":1.726},"min":{"1min":0.27400000000000002,"5min":0.19,"15min":0.19},"max":{"1min":7.8209999999999997,"5min":47.057000000000002,"15min":47.057000000000002},"last":0.21199999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.8839999999999999,"15min":2.0430000000000001},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.218},"max":{"1min":5.7249999999999996,"5min":54.752000000000002,"15min":54.752000000000002},"last":1.476}]},{"osd":2,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":2.9649999999999999,"15min":2.496},"min":{"1min":0.39900000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.1669999999999998,"5min":47.097999999999999,"15min":70.594999999999999},"last":1.6419999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":2.452,"15min":2.0329999999999999},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":3.0430000000000001,"5min":46.997,"15min":62.991999999999997},"last":0.255}]},{"osd":4,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.6509999999999998,"15min":1.978},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.411,"5min":72.653000000000006,"15min":72.653000000000006},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.80400000000000005,"5min":2.4430000000000001,"15min":2.0409999999999999},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":1.752,"5min":72.706999999999994,"15min":72.706999999999994},"last":0.33100000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.845,"15min":1.734},"min":{"1min":0.42699999999999999,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":2.3820000000000001,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.9810000000000001,"15min":1.8380000000000001},"min":{"1min":0.48699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.363,"5min":9.4619999999999997,"15min":11.754},"last":0.36899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.97,"15min":1.708},"min":{"1min":0.48399999999999999,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":3.0880000000000001,"5min":15.122999999999999,"15min":15.122999999999999},"last":1.5649999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.9790000000000001,"15min":1.653},"min":{"1min":0.38100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.964,"5min":11.728999999999999,"15min":11.728999999999999},"last":3.1629999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.0699999999999998,"15min":1.837},"min":{"1min":0.51900000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.3069999999999999,"5min":8.6059999999999999,"15min":18.695},"last":1.042},{"interface":"front","average":{"1min":1.1839999999999999,"5min":2.0880000000000001,"15min":1.9419999999999999},"min":{"1min":0.47099999999999997,"5min":0.43099999999999999,"15min":0.39900000000000002},"max":{"1min":3.1429999999999998,"5min":13.211,"15min":13.211},"last":1.595}]}]},{"osd":4,"up_from":30,"seq":128849019338,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1625436,"kb_used_data":1324116,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92119716,"statfs":{"total":95995035648,"available":94330589184,"internally_reserved":0,"allocated":1355894784,"data_stored":1355345239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":2.6739999999999999,"15min":2.3700000000000001},"min":{"1min":0.58599999999999997,"5min":0.254,"15min":0.254},"max":{"1min":15.282999999999999,"5min":115.59699999999999,"15min":115.59699999999999},"last":2.1840000000000002},{"interface":"front","average":{"1min":1.825,"5min":2.7869999999999999,"15min":2.694},"min":{"1min":0.66400000000000003,"5min":0.28000000000000003,"15min":0.27700000000000002},"max":{"1min":15.242000000000001,"5min":115.65000000000001,"15min":115.65000000000001},"last":2.133}]},{"osd":1,"last update":"Mon Oct 9 06:34:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.47,"15min":2.2639999999999998},"min":{"1min":0.45100000000000001,"5min":0.29199999999999998,"15min":0.27600000000000002},"max":{"1min":17.033000000000001,"5min":17.033000000000001,"15min":179.47300000000001},"last":3.5710000000000002},{"interface":"front","average":{"1min":2.1360000000000001,"5min":1.6859999999999999,"15min":2.6699999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.28499999999999998},"max":{"1min":17.015999999999998,"5min":17.015999999999998,"15min":179.447},"last":3.5819999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.6140000000000001,"15min":2.1309999999999998},"min":{"1min":0.57199999999999995,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":1.2689999999999999,"5min":14.705,"15min":61.783999999999999},"last":2.056},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.5760000000000001,"15min":2.2599999999999998},"min":{"1min":0.55400000000000005,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":1.3069999999999999,"5min":9.7759999999999998,"15min":61.551000000000002},"last":2.1560000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:18 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":2.2250000000000001,"15min":2.3460000000000001},"min":{"1min":0.40300000000000002,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":7.2530000000000001,"5min":12.081,"15min":91.361999999999995},"last":2.2650000000000001},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.9950000000000001,"15min":1.9319999999999999},"min":{"1min":0.47999999999999998,"5min":0.36699999999999999,"15min":0.313},"max":{"1min":7.1870000000000003,"5min":16.257000000000001,"15min":89.676000000000002},"last":2.2559999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:33 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.90000000000000002,"15min":1.2949999999999999},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4550000000000001,"5min":11.737,"15min":12.242000000000001},"last":2.226},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.234,"15min":1.708},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.26500000000000001},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":20.113},"last":2.2799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.288,"15min":1.5820000000000001},"min":{"1min":0.54700000000000004,"5min":0.35399999999999998,"15min":0.29499999999999998},"max":{"1min":1.609,"5min":8.1110000000000007,"15min":18.010000000000002},"last":2.0950000000000002},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.2809999999999999,"15min":1.4319999999999999},"min":{"1min":0.39100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4199999999999999,"5min":11.782,"15min":13.913},"last":2.3010000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.4419999999999999,"15min":1.776},"min":{"1min":0.73399999999999999,"5min":0.42399999999999999,"15min":0.371},"max":{"1min":2.6480000000000001,"5min":12.779,"15min":15.294},"last":2.2029999999999998},{"interface":"front","average":{"1min":1.006,"5min":1.226,"15min":1.7470000000000001},"min":{"1min":0.432,"5min":0.317,"15min":0.317},"max":{"1min":1.6399999999999999,"5min":9.1370000000000005,"15min":14.896000000000001},"last":2.3130000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823105,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2435528,"kb_used_data":2126016,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91309624,"statfs":{"total":95995035648,"available":93501054976,"internally_reserved":0,"allocated":2177040384,"data_stored":2176506452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":2.9169999999999998,"15min":2.6989999999999998},"min":{"1min":0.56999999999999995,"5min":0.254,"15min":0.247},"max":{"1min":1.0800000000000001,"5min":83.197000000000003,"15min":83.197000000000003},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.7360000000000002,"15min":2.4209999999999998},"min":{"1min":0.56899999999999995,"5min":0.29099999999999998,"15min":0.28100000000000003},"max":{"1min":1.3160000000000001,"5min":83.563999999999993,"15min":83.563999999999993},"last":1.6279999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.3939999999999999,"15min":2.4329999999999998},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":3.694,"5min":5.9299999999999997,"15min":161.71199999999999},"last":1.5640000000000001},{"interface":"front","average":{"1min":1.98,"5min":1.698,"15min":2.7080000000000002},"min":{"1min":0.624,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":11.753,"5min":11.753,"15min":162.50200000000001},"last":0.53200000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":2.0979999999999999,"15min":2.2890000000000001},"min":{"1min":0.67000000000000004,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":1.1970000000000001,"5min":54.981999999999999,"15min":54.981999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.052,"5min":2.2010000000000001,"15min":2.2389999999999999},"min":{"1min":0.64500000000000002,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":2.149,"5min":55.008000000000003,"15min":59.082999999999998},"last":1.9890000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":3.3839999999999999,"15min":2.8769999999999998},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":5.0670000000000002,"5min":84.174000000000007,"15min":84.174000000000007},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.153,"5min":3.4750000000000001,"15min":2.7210000000000001},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.786,"5min":83.114999999999995,"15min":83.114999999999995},"last":1.7110000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.288,"15min":1.8740000000000001},"min":{"1min":0.375,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.0990000000000002,"5min":8.8879999999999999,"15min":17.667999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.222,"15min":1.6080000000000001},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.308},"max":{"1min":5.0919999999999996,"5min":5.3040000000000003,"15min":16.181999999999999},"last":0.36899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.3320000000000001,"15min":1.7270000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.133,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.504},{"interface":"front","average":{"1min":1.292,"5min":1.292,"15min":1.631},"min":{"1min":0.45500000000000002,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":5.1180000000000003,"5min":9.3450000000000006,"15min":12.365},"last":0.66300000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:33:50 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.115,"15min":1.708},"min":{"1min":0.99299999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.4590000000000001,"5min":5.7039999999999997,"15min":13.047000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.2350000000000001,"15min":1.526},"min":{"1min":0.91900000000000004,"5min":0.41599999999999998,"15min":0.36799999999999999},"max":{"1min":1.4379999999999999,"5min":5.5919999999999996,"15min":11.981},"last":1.52}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1644232704,"data_stored":1644232704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1856634880,"data_stored":1856634880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1554448384,"data_stored":1554448384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1572012032,"data_stored":1572012032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1376256000,"data_stored":1376256000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2200567808,"data_stored":2200567808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1274150912,"data_stored":1274150912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1375993856,"data_stored":1375993856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:45.146 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:34:45.146 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:34:45.146 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-09T06:34:45.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:45.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:45.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:45.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:45.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:45.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:45.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:45.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:45.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:45.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:45.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:45.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:45.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 73498 73482 93.6867 66.5 0.007766 0.0106678 2023-10-09T06:34:45.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:45.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:45.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:45.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:34:45.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:46.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:46.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:46.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:46.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:46.256 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:46 smithi155 bash[59495]: cluster 2023-10-09T06:34:45.132001+0000 mgr.y (mgr.25723) 900 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:46.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:46.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:46.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:46.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:46.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:46.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:46.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:46.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:46.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:46 smithi125 bash[77299]: cluster 2023-10-09T06:34:45.132001+0000 mgr.y (mgr.25723) 900 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:46.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:46 smithi125 bash[80002]: cluster 2023-10-09T06:34:45.132001+0000 mgr.y (mgr.25723) 900 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:46.649 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:46.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:46.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:46.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 74526 74510 93.0981 64.25 0.0136886 0.0107351 2023-10-09T06:34:46.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:46.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:46.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:46.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:47.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:47.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:47.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:47.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:34:47.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:47.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:47.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:47.366 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:47.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:47.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:47.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:47.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:47.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:47.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:47.801 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 75518 75502 92.4883 62 0.0110824 0.0108057 2023-10-09T06:34:47.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:47.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:47.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:48.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:48.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:48.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:48.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:48.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:48.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:48.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:48.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:48.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:48.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:48 smithi125 bash[80002]: cluster 2023-10-09T06:34:47.132814+0000 mgr.y (mgr.25723) 901 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:34:48.563 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:48 smithi125 bash[77299]: cluster 2023-10-09T06:34:47.132814+0000 mgr.y (mgr.25723) 901 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:34:48.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:48.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:48 smithi155 bash[59495]: cluster 2023-10-09T06:34:47.132814+0000 mgr.y (mgr.25723) 901 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:34:48.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:48.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:48.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 15 76519 76504 91.9141 62.625 0.0154901 0.0108742 2023-10-09T06:34:48.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:48.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:48.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:48.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:49.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:49.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:49.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.254 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:49 smithi125 bash[77299]: audit 2023-10-09T06:34:48.589344+0000 mon.a (mon.0) 1106 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:49.254 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:49 smithi125 bash[80002]: audit 2023-10-09T06:34:48.589344+0000 mon.a (mon.0) 1106 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:49.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:49.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:49 smithi155 bash[59495]: audit 2023-10-09T06:34:48.589344+0000 mon.a (mon.0) 1106 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:49.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:49.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:49.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:49.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:49.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:49.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:49.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 77543 77527 91.3862 63.9375 0.00977753 0.0109367 2023-10-09T06:34:49.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:49.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.931 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:49.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:49.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:49.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:49.993 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:49.993 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:50.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:50.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:50.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:50.201 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:50.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:50.318 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:50 smithi125 bash[77299]: cluster 2023-10-09T06:34:49.133429+0000 mgr.y (mgr.25723) 902 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:34:50.318 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:50 smithi125 bash[80002]: cluster 2023-10-09T06:34:49.133429+0000 mgr.y (mgr.25723) 902 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:34:50.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:50.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:50.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:50.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:50.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:50.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:50 smithi155 bash[59495]: cluster 2023-10-09T06:34:49.133429+0000 mgr.y (mgr.25723) 902 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:34:50.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:50.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:50.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 78546 78530 90.8548 62.6875 0.00831801 0.0110001 2023-10-09T06:34:50.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:50.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:50.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:50.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.010 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:51.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:51.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:51.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:51.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:51.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:51.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:51.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:34:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:34:51] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:34:51.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:51.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:51.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:51.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:51.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:51.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:51.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 79512 79496 90.3007 60.375 0.105468 0.0110554 2023-10-09T06:34:51.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:51.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:51.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:51.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:52.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:52.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:52.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:52.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:52.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:52.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:52.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:52.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:52 smithi125 bash[77299]: cluster 2023-10-09T06:34:51.134076+0000 mgr.y (mgr.25723) 903 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:34:52.464 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:52 smithi125 bash[80002]: cluster 2023-10-09T06:34:51.134076+0000 mgr.y (mgr.25723) 903 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:34:52.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:52.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:52.596 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:52.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:52 smithi155 bash[59495]: cluster 2023-10-09T06:34:51.134076+0000 mgr.y (mgr.25723) 903 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T06:34:52.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:52.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:52.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 80494 80478 89.7842 61.375 0.00589891 0.0111319 2023-10-09T06:34:52.852 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:52.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:52.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:52.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:53.012 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:53.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:53.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:53.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:53.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:53.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:53.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:53.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:53.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:53.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:53.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:53.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:53.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 81537 81521 89.3527 65.1875 0.0238169 0.0111858 2023-10-09T06:34:53.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:53.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:53.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:53.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:54.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:54.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:54.269 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:54 smithi125 bash[77299]: cluster 2023-10-09T06:34:53.134701+0000 mgr.y (mgr.25723) 904 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:54.269 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:54 smithi125 bash[80002]: cluster 2023-10-09T06:34:53.134701+0000 mgr.y (mgr.25723) 904 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:54.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:54.333 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:54 smithi155 bash[59495]: cluster 2023-10-09T06:34:53.134701+0000 mgr.y (mgr.25723) 904 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:34:54.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:54.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:54.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:54.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:54.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:54.736 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:54.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 82557 82541 88.9112 63.75 0.0126154 0.0112416 2023-10-09T06:34:54.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:54.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:54.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:54.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:54.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:55.013 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:55.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:55.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:55.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:55.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:55.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:55.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:55.508 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:83231 8.2 8:4cdc2826:::benchmark_data_smithi125_176369_object83230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:34:55.491714+0000", 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.014383183000000001, 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014422836, 2023-10-09T06:34:55.509 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83231 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491714+0000", 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491714+0000", 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999952 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491709+0000", 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9762000000000002e-05 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:34:55.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491759+0000", 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5952000000000002e-05 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491785+0000", 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0169999999999999e-05 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.491795+0000", 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040773299999999997 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:34:55.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.495872+0000", 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015139 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.496024+0000", 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012723199999999999 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.515 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.496151+0000", 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082600429999999999 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.504411+0000", 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2734000000000002e-05 2023-10-09T06:34:55.516 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.504444+0000", 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012729646000000001 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:83233 8.2 8:42f0dce5:::benchmark_data_smithi125_176369_object83232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:34:55.492529+0000", 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.013568353, 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013710742999999999, 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:34:55.518 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:34:55.519 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:34:55.519 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 83233 2023-10-09T06:34:55.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.519 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:34:55.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:34:55.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492529+0000", 2023-10-09T06:34:55.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492529+0000", 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492528+0000", 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116335 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492644+0000", 2023-10-09T06:34:55.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4711e-05 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492659+0000", 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1939999999999998e-06 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.492667+0000", 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090600980000000008 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.501727+0000", 2023-10-09T06:34:55.524 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8892000000000001e-05 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.501826+0000", 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000107329 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:34:55.525 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.501933+0000", 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026477269999999999 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:34:55.504581+0000", 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012051948999999999 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:34:55.526 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:34:55.527 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:34:55.527 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:34:55.527 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 2 2023-10-09T06:34:55.527 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:34:55.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:55.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:55.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:55.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:55.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:55.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:55.802 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 83493 83477 88.3958 58.5 0.0111549 0.0113042 2023-10-09T06:34:55.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:55.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:55.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:55.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:56.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:56.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:56.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:56.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:56.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:56.350 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:56 smithi155 bash[59495]: cluster 2023-10-09T06:34:55.135298+0000 mgr.y (mgr.25723) 905 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:56.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:56.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:56 smithi125 bash[77299]: cluster 2023-10-09T06:34:55.135298+0000 mgr.y (mgr.25723) 905 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:56.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:56 smithi125 bash[80002]: cluster 2023-10-09T06:34:55.135298+0000 mgr.y (mgr.25723) 905 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:34:56.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:56.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:56.683 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:56.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:34:56.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:56.803 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:34:56.796728+0000 min lat: 0.00131043 max lat: 0.352992 avg lat: 0.0113654 2023-10-09T06:34:56.803 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:34:56.803 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 84473 84457 87.9434 61.25 0.0223003 0.0113654 2023-10-09T06:34:56.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:56.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:56.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:56.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:56.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:57.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:57.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:57.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:57.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:57.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:57.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:57.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:57.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:57.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:57.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:57.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:57.778 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:57.805 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 85443 85427 87.4932 60.625 0.019541 0.0114239 2023-10-09T06:34:57.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:57.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:57.909 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:57.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:57.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:57.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:58.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:58.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:58.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:58.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:58.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:58.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.315 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:58 smithi155 bash[59495]: cluster 2023-10-09T06:34:57.135757+0000 mgr.y (mgr.25723) 906 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-10-09T06:34:58.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:58.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:58.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:58.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:58.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:58 smithi125 bash[77299]: cluster 2023-10-09T06:34:57.135757+0000 mgr.y (mgr.25723) 906 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-10-09T06:34:58.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:58 smithi125 bash[80002]: cluster 2023-10-09T06:34:57.135757+0000 mgr.y (mgr.25723) 906 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2023-10-09T06:34:58.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:58.804 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 86284 86268 86.9298 52.5625 0.0119677 0.0114977 2023-10-09T06:34:58.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:58.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:58.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:59.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:59.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:59.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:59.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:59.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:59.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:59.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:59.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:59.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:59.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:59.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:59.805 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 87126 87110 86.3854 52.625 0.00473688 0.0115483 2023-10-09T06:34:59.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:59.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:59.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:59.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:59.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:34:59.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:00.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:00.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:34:59 smithi155 bash[59495]: cluster 2023-10-09T06:34:59.136197+0000 mgr.y (mgr.25723) 907 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:35:00.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:34:59 smithi125 bash[77299]: cluster 2023-10-09T06:34:59.136197+0000 mgr.y (mgr.25723) 907 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:35:00.185 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:34:59 smithi125 bash[80002]: cluster 2023-10-09T06:34:59.136197+0000 mgr.y (mgr.25723) 907 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:35:00.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:00.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:00.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:00.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:00.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:00.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:00.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:00.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:00.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:00.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:00.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:00.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:00.805 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 87794 87778 85.6881 41.75 0.00452256 0.0116398 2023-10-09T06:35:00.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:00.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:00.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:01.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:01.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:01.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:01.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:01.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:01.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:01.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:01.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:01] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:35:01.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:01.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:01.539 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:01.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:01.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:01.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:01.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:01.805 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 88710 88694 85.2506 57.25 0.0144849 0.0117238 2023-10-09T06:35:01.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:01.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:01.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:01.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:01.997 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:02.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:02.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:02.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:02.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:02.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:02.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:02.345 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:35:02.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:02.386 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:35:02.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:02 smithi125 bash[77299]: cluster 2023-10-09T06:35:01.136871+0000 mgr.y (mgr.25723) 908 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T06:35:02.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:02 smithi125 bash[80002]: cluster 2023-10-09T06:35:01.136871+0000 mgr.y (mgr.25723) 908 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T06:35:02.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:02 smithi155 bash[59495]: cluster 2023-10-09T06:35:01.136871+0000 mgr.y (mgr.25723) 908 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-10-09T06:35:02.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:02.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:02.427 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:35:02.475 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:35:02.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:02.526 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:35:02.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:02.564 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:35:02.597 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:35:02.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:02.646 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:35:02.657 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.657 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.658 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:35:02.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:02.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:02.806 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 89462 89446 84.6692 47 0.00648237 0.0118044 2023-10-09T06:35:02.815 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:02.828 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:35:02.838 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:35:02.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:02.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:02.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:03.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:03.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:03.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:03.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:03 smithi125 bash[77299]: audit 2023-10-09T06:35:02.286046+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.125:0/4007683733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:03.202 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:03 smithi125 bash[77299]: audit 2023-10-09T06:35:02.486998+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:03.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:03 smithi125 bash[80002]: audit 2023-10-09T06:35:02.286046+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.125:0/4007683733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:03.203 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:03 smithi125 bash[80002]: audit 2023-10-09T06:35:02.486998+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:03.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:03.222 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:35:03.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:03.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:03 smithi155 bash[59495]: audit 2023-10-09T06:35:02.286046+0000 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.125:0/4007683733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:03.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:03 smithi155 bash[59495]: audit 2023-10-09T06:35:02.486998+0000 mon.a (mon.0) 1107 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:03.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:03.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:03.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:03.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:03.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:03.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:03.806 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 90349 90333 84.2329 55.4375 0.0129691 0.0118667 2023-10-09T06:35:03.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:03.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:03.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:04.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:04.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:04.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:04.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:04.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: audit 2023-10-09T06:35:02.808386+0000 mgr.y (mgr.25723) 909 : audit 0 from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.138886+0000 mgr.y (mgr.25723) 910 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.193819+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.193889+0000 mon.a (mon.0) 1109 : cluster 1 Cluster is now healthy 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: audit 2023-10-09T06:35:03.210839+0000 mon.a (mon.0) 1110 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:04.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.210929+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:35:04.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.211492+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: cluster 2023-10-09T06:35:03.220008+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: audit 2023-10-09T06:35:03.598816+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:04.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:04 smithi155 bash[59495]: audit 2023-10-09T06:35:03.605365+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:04.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:04.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:04.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:04.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: audit 2023-10-09T06:35:02.808386+0000 mgr.y (mgr.25723) 909 : audit 0 from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.138886+0000 mgr.y (mgr.25723) 910 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.193819+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.193889+0000 mon.a (mon.0) 1109 : cluster 1 Cluster is now healthy 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: audit 2023-10-09T06:35:03.210839+0000 mon.a (mon.0) 1110 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.210929+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.211492+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: cluster 2023-10-09T06:35:03.220008+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: audit 2023-10-09T06:35:03.598816+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:04.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[77299]: audit 2023-10-09T06:35:03.605365+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: audit 2023-10-09T06:35:02.808386+0000 mgr.y (mgr.25723) 909 : audit 0 from='client.35541 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.138886+0000 mgr.y (mgr.25723) 910 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.193819+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.193889+0000 mon.a (mon.0) 1109 : cluster 1 Cluster is now healthy 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: audit 2023-10-09T06:35:03.210839+0000 mon.a (mon.0) 1110 : audit 1 from='client.? 172.21.15.125:0/1205283148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.210929+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.211492+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: cluster 2023-10-09T06:35:03.220008+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e199 with expected crc 2023-10-09T06:35:04.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: audit 2023-10-09T06:35:03.598816+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:04.469 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:04 smithi125 bash[80002]: audit 2023-10-09T06:35:03.605365+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:04.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:04.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:04.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:04.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:04.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:04.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:04.806 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 91148 91132 83.7286 49.9375 0.0144963 0.0119378 2023-10-09T06:35:04.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:04.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:04.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:05.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:05.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:05.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:05.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:05.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:05.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:05.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:05.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:05.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:05.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:05.736 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":918,"stamp":"2023-10-09T06:35:01.136367+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5637347251,"num_objects":85974,"num_object_clones":0,"num_object_copies":257922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85974,"num_whiteouts":0,"num_read":6289,"num_read_kb":13748,"num_write":87052,"num_write_kb":5515703,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":165,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87116,"ondisk_log_size":87116,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19423728,"kb_used_data":16443248,"kb_used_omap":30,"kb_used_meta":2980385,"kb_avail":730537488,"statfs":{"total":767960285184,"available":748070387712,"internally_reserved":0,"allocated":16837885952,"data_stored":16832419313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":3051914351},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,1,10,3,8,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":768606208,"num_objects":11728,"num_object_clones":0,"num_object_copies":35184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11728,"num_write_kb":750592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003353"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223900+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:34:08.223900+0000","last_peered":"2023-10-09T06:34:08.223900+0000","last_clean":"2023-10-09T06:34:08.223900+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:34:08.223900+0000","last_undegraded":"2023-10-09T06:34:08.223900+0000","last_fullsized":"2023-10-09T06:34:08.223900+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.926183+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:34:39.926183+0000","last_peered":"2023-10-09T06:34:39.926183+0000","last_clean":"2023-10-09T06:34:39.926183+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:34:39.926183+0000","last_undegraded":"2023-10-09T06:34:39.926183+0000","last_fullsized":"2023-10-09T06:34:39.926183+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213497+0000","last_change":"2023-10-09T06:14:42.685575+0000","last_active":"2023-10-09T06:34:08.213497+0000","last_peered":"2023-10-09T06:34:08.213497+0000","last_clean":"2023-10-09T06:34:08.213497+0000","last_became_active":"2023-10-09T06:14:42.685241+0000","last_became_peered":"2023-10-09T06:14:42.685241+0000","last_unstale":"2023-10-09T06:34:08.213497+0000","last_undegraded":"2023-10-09T06:34:08.213497+0000","last_fullsized":"2023-10-09T06:34:08.213497+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:09.194276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.216987+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:34:08.216987+0000","last_peered":"2023-10-09T06:34:08.216987+0000","last_clean":"2023-10-09T06:34:08.216987+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:34:08.216987+0000","last_undegraded":"2023-10-09T06:34:08.216987+0000","last_fullsized":"2023-10-09T06:34:08.216987+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":190,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.219172+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:34:08.219172+0000","last_peered":"2023-10-09T06:34:08.219172+0000","last_clean":"2023-10-09T06:34:08.219172+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:34:08.219172+0000","last_undegraded":"2023-10-09T06:34:08.219172+0000","last_fullsized":"2023-10-09T06:34:08.219172+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":38,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.222084+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:34:08.222084+0000","last_peered":"2023-10-09T06:34:08.222084+0000","last_clean":"2023-10-09T06:34:08.222084+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:34:08.222084+0000","last_undegraded":"2023-10-09T06:34:08.222084+0000","last_fullsized":"2023-10-09T06:34:08.222084+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":154,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.916378+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:34:39.916378+0000","last_peered":"2023-10-09T06:34:39.916378+0000","last_clean":"2023-10-09T06:34:39.916378+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:34:39.916378+0000","last_undegraded":"2023-10-09T06:34:39.916378+0000","last_fullsized":"2023-10-09T06:34:39.916378+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":205,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.219214+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:34:08.219214+0000","last_peered":"2023-10-09T06:34:08.219214+0000","last_clean":"2023-10-09T06:34:08.219214+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:34:08.219214+0000","last_undegraded":"2023-10-09T06:34:08.219214+0000","last_fullsized":"2023-10-09T06:34:08.219214+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":181,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225237+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:34:08.225237+0000","last_peered":"2023-10-09T06:34:08.225237+0000","last_clean":"2023-10-09T06:34:08.225237+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:34:08.225237+0000","last_undegraded":"2023-10-09T06:34:08.225237+0000","last_fullsized":"2023-10-09T06:34:08.225237+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":200,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.218473+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:34:08.218473+0000","last_peered":"2023-10-09T06:34:08.218473+0000","last_clean":"2023-10-09T06:34:08.218473+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:34:08.218473+0000","last_undegraded":"2023-10-09T06:34:08.218473+0000","last_fullsized":"2023-10-09T06:34:08.218473+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224826+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:34:08.224826+0000","last_peered":"2023-10-09T06:34:08.224826+0000","last_clean":"2023-10-09T06:34:08.224826+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:34:08.224826+0000","last_undegraded":"2023-10-09T06:34:08.224826+0000","last_fullsized":"2023-10-09T06:34:08.224826+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":169,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.942694+0000","last_change":"2023-10-09T06:28:31.015091+0000","last_active":"2023-10-09T06:34:39.942694+0000","last_peered":"2023-10-09T06:34:39.942694+0000","last_clean":"2023-10-09T06:34:39.942694+0000","last_became_active":"2023-10-09T06:28:31.014673+0000","last_became_peered":"2023-10-09T06:28:31.014673+0000","last_unstale":"2023-10-09T06:34:39.942694+0000","last_undegraded":"2023-10-09T06:34:39.942694+0000","last_fullsized":"2023-10-09T06:34:39.942694+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:13:32.834463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":246,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224177+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:34:08.224177+0000","last_peered":"2023-10-09T06:34:08.224177+0000","last_clean":"2023-10-09T06:34:08.224177+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:34:08.224177+0000","last_undegraded":"2023-10-09T06:34:08.224177+0000","last_fullsized":"2023-10-09T06:34:08.224177+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224476+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:34:08.224476+0000","last_peered":"2023-10-09T06:34:08.224476+0000","last_clean":"2023-10-09T06:34:08.224476+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:34:08.224476+0000","last_undegraded":"2023-10-09T06:34:08.224476+0000","last_fullsized":"2023-10-09T06:34:08.224476+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":96,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224640+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:34:08.224640+0000","last_peered":"2023-10-09T06:34:08.224640+0000","last_clean":"2023-10-09T06:34:08.224640+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:34:08.224640+0000","last_undegraded":"2023-10-09T06:34:08.224640+0000","last_fullsized":"2023-10-09T06:34:08.224640+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":192,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.220476+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:34:08.220476+0000","last_peered":"2023-10-09T06:34:08.220476+0000","last_clean":"2023-10-09T06:34:08.220476+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:34:08.220476+0000","last_undegraded":"2023-10-09T06:34:08.220476+0000","last_fullsized":"2023-10-09T06:34:08.220476+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.718734+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:34:13.718734+0000","last_peered":"2023-10-09T06:34:13.718734+0000","last_clean":"2023-10-09T06:34:13.718734+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:34:13.718734+0000","last_undegraded":"2023-10-09T06:34:13.718734+0000","last_fullsized":"2023-10-09T06:34:13.718734+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225252+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:34:08.225252+0000","last_peered":"2023-10-09T06:34:08.225252+0000","last_clean":"2023-10-09T06:34:08.225252+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:34:08.225252+0000","last_undegraded":"2023-10-09T06:34:08.225252+0000","last_fullsized":"2023-10-09T06:34:08.225252+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":108,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224387+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:34:08.224387+0000","last_peered":"2023-10-09T06:34:08.224387+0000","last_clean":"2023-10-09T06:34:08.224387+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:34:08.224387+0000","last_undegraded":"2023-10-09T06:34:08.224387+0000","last_fullsized":"2023-10-09T06:34:08.224387+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.947812+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:34:39.947812+0000","last_peered":"2023-10-09T06:34:39.947812+0000","last_clean":"2023-10-09T06:34:39.947812+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:34:39.947812+0000","last_undegraded":"2023-10-09T06:34:39.947812+0000","last_fullsized":"2023-10-09T06:34:39.947812+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":48,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.218141+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:34:08.218141+0000","last_peered":"2023-10-09T06:34:08.218141+0000","last_clean":"2023-10-09T06:34:08.218141+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:34:08.218141+0000","last_undegraded":"2023-10-09T06:34:08.218141+0000","last_fullsized":"2023-10-09T06:34:08.218141+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":205,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.226627+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:34:08.226627+0000","last_peered":"2023-10-09T06:34:08.226627+0000","last_clean":"2023-10-09T06:34:08.226627+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:34:08.226627+0000","last_undegraded":"2023-10-09T06:34:08.226627+0000","last_fullsized":"2023-10-09T06:34:08.226627+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"198'5476","reported_seq":5490,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.657978+0000","last_change":"2023-10-09T06:33:33.324346+0000","last_active":"2023-10-09T06:34:59.657978+0000","last_peered":"2023-10-09T06:34:59.657978+0000","last_clean":"2023-10-09T06:34:59.657978+0000","last_became_active":"2023-10-09T06:33:33.323046+0000","last_became_peered":"2023-10-09T06:33:33.323046+0000","last_unstale":"2023-10-09T06:34:59.657978+0000","last_undegraded":"2023-10-09T06:34:59.657978+0000","last_fullsized":"2023-10-09T06:34:59.657978+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358875136,"num_objects":5476,"num_object_clones":0,"num_object_copies":16428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5476,"num_write_kb":350464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225010+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:34:08.225010+0000","last_peered":"2023-10-09T06:34:08.225010+0000","last_clean":"2023-10-09T06:34:08.225010+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:34:08.225010+0000","last_undegraded":"2023-10-09T06:34:08.225010+0000","last_fullsized":"2023-10-09T06:34:08.225010+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":183,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225553+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:34:08.225553+0000","last_peered":"2023-10-09T06:34:08.225553+0000","last_clean":"2023-10-09T06:34:08.225553+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:34:08.225553+0000","last_undegraded":"2023-10-09T06:34:08.225553+0000","last_fullsized":"2023-10-09T06:34:08.225553+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"198'5416","reported_seq":5430,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.689866+0000","last_change":"2023-10-09T06:33:33.593835+0000","last_active":"2023-10-09T06:34:59.689866+0000","last_peered":"2023-10-09T06:34:59.689866+0000","last_clean":"2023-10-09T06:34:59.689866+0000","last_became_active":"2023-10-09T06:33:33.593363+0000","last_became_peered":"2023-10-09T06:33:33.593363+0000","last_unstale":"2023-10-09T06:34:59.689866+0000","last_undegraded":"2023-10-09T06:34:59.689866+0000","last_fullsized":"2023-10-09T06:34:59.689866+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5416,"log_dups_size":0,"ondisk_log_size":5416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354942976,"num_objects":5416,"num_object_clones":0,"num_object_copies":16248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5416,"num_write_kb":346624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":94,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225233+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:34:08.225233+0000","last_peered":"2023-10-09T06:34:08.225233+0000","last_clean":"2023-10-09T06:34:08.225233+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:34:08.225233+0000","last_undegraded":"2023-10-09T06:34:08.225233+0000","last_fullsized":"2023-10-09T06:34:08.225233+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223894+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:34:08.223894+0000","last_peered":"2023-10-09T06:34:08.223894+0000","last_clean":"2023-10-09T06:34:08.223894+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:34:08.223894+0000","last_undegraded":"2023-10-09T06:34:08.223894+0000","last_fullsized":"2023-10-09T06:34:08.223894+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"198'5382","reported_seq":5397,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:58.796171+0000","last_change":"2023-10-09T06:33:33.587458+0000","last_active":"2023-10-09T06:34:58.796171+0000","last_peered":"2023-10-09T06:34:58.796171+0000","last_clean":"2023-10-09T06:34:58.796171+0000","last_became_active":"2023-10-09T06:33:33.586581+0000","last_became_peered":"2023-10-09T06:33:33.586581+0000","last_unstale":"2023-10-09T06:34:58.796171+0000","last_undegraded":"2023-10-09T06:34:58.796171+0000","last_fullsized":"2023-10-09T06:34:58.796171+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.486216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714752,"num_objects":5382,"num_object_clones":0,"num_object_copies":16146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5382,"num_write_kb":344448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":120,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224627+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:34:08.224627+0000","last_peered":"2023-10-09T06:34:08.224627+0000","last_clean":"2023-10-09T06:34:08.224627+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:34:08.224627+0000","last_undegraded":"2023-10-09T06:34:08.224627+0000","last_fullsized":"2023-10-09T06:34:08.224627+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":38,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.221523+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:34:08.221523+0000","last_peered":"2023-10-09T06:34:08.221523+0000","last_clean":"2023-10-09T06:34:08.221523+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:34:08.221523+0000","last_undegraded":"2023-10-09T06:34:08.221523+0000","last_fullsized":"2023-10-09T06:34:08.221523+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"198'5309","reported_seq":5323,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:57.458303+0000","last_change":"2023-10-09T06:33:33.319454+0000","last_active":"2023-10-09T06:34:57.458303+0000","last_peered":"2023-10-09T06:34:57.458303+0000","last_clean":"2023-10-09T06:34:57.458303+0000","last_became_active":"2023-10-09T06:33:33.318032+0000","last_became_peered":"2023-10-09T06:33:33.318032+0000","last_unstale":"2023-10-09T06:34:57.458303+0000","last_undegraded":"2023-10-09T06:34:57.458303+0000","last_fullsized":"2023-10-09T06:34:57.458303+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:46.906949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5309,"num_write_kb":339776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225551+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:34:08.225551+0000","last_peered":"2023-10-09T06:34:08.225551+0000","last_clean":"2023-10-09T06:34:08.225551+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:34:08.225551+0000","last_undegraded":"2023-10-09T06:34:08.225551+0000","last_fullsized":"2023-10-09T06:34:08.225551+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215382+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:34:08.215382+0000","last_peered":"2023-10-09T06:34:08.215382+0000","last_clean":"2023-10-09T06:34:08.215382+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:34:08.215382+0000","last_undegraded":"2023-10-09T06:34:08.215382+0000","last_fullsized":"2023-10-09T06:34:08.215382+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"198'5320","reported_seq":5335,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:56.557022+0000","last_change":"2023-10-09T06:33:33.320528+0000","last_active":"2023-10-09T06:34:56.557022+0000","last_peered":"2023-10-09T06:34:56.557022+0000","last_clean":"2023-10-09T06:34:56.557022+0000","last_became_active":"2023-10-09T06:33:33.320042+0000","last_became_peered":"2023-10-09T06:33:33.320042+0000","last_unstale":"2023-10-09T06:34:56.557022+0000","last_undegraded":"2023-10-09T06:34:56.557022+0000","last_fullsized":"2023-10-09T06:34:56.557022+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5320,"log_dups_size":0,"ondisk_log_size":5320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.099867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348651520,"num_objects":5320,"num_object_clones":0,"num_object_copies":15960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5320,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.926111+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:34:39.926111+0000","last_peered":"2023-10-09T06:34:39.926111+0000","last_clean":"2023-10-09T06:34:39.926111+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:34:39.926111+0000","last_undegraded":"2023-10-09T06:34:39.926111+0000","last_fullsized":"2023-10-09T06:34:39.926111+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224166+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:34:08.224166+0000","last_peered":"2023-10-09T06:34:08.224166+0000","last_clean":"2023-10-09T06:34:08.224166+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:34:08.224166+0000","last_undegraded":"2023-10-09T06:34:08.224166+0000","last_fullsized":"2023-10-09T06:34:08.224166+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"198'5393","reported_seq":5408,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:57.456492+0000","last_change":"2023-10-09T06:33:33.321209+0000","last_active":"2023-10-09T06:34:57.456492+0000","last_peered":"2023-10-09T06:34:57.456492+0000","last_clean":"2023-10-09T06:34:57.456492+0000","last_became_active":"2023-10-09T06:33:33.320866+0000","last_became_peered":"2023-10-09T06:33:33.320866+0000","last_unstale":"2023-10-09T06:34:57.456492+0000","last_undegraded":"2023-10-09T06:34:57.456492+0000","last_fullsized":"2023-10-09T06:34:57.456492+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:34.914529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353435648,"num_objects":5393,"num_object_clones":0,"num_object_copies":16179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5393,"num_write_kb":345152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224073+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:34:08.224073+0000","last_peered":"2023-10-09T06:34:08.224073+0000","last_clean":"2023-10-09T06:34:08.224073+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:34:08.224073+0000","last_undegraded":"2023-10-09T06:34:08.224073+0000","last_fullsized":"2023-10-09T06:34:08.224073+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224264+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:34:08.224264+0000","last_peered":"2023-10-09T06:34:08.224264+0000","last_clean":"2023-10-09T06:34:08.224264+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:34:08.224264+0000","last_undegraded":"2023-10-09T06:34:08.224264+0000","last_fullsized":"2023-10-09T06:34:08.224264+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"198'5505","reported_seq":5519,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.658496+0000","last_change":"2023-10-09T06:33:33.244614+0000","last_active":"2023-10-09T06:34:59.658496+0000","last_peered":"2023-10-09T06:34:59.658496+0000","last_clean":"2023-10-09T06:34:59.658496+0000","last_became_active":"2023-10-09T06:33:33.243719+0000","last_became_peered":"2023-10-09T06:33:33.243719+0000","last_unstale":"2023-10-09T06:34:59.658496+0000","last_undegraded":"2023-10-09T06:34:59.658496+0000","last_fullsized":"2023-10-09T06:34:59.658496+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5505,"log_dups_size":0,"ondisk_log_size":5505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360775680,"num_objects":5505,"num_object_clones":0,"num_object_copies":16515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5505,"num_write_kb":352320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":181,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223583+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:34:08.223583+0000","last_peered":"2023-10-09T06:34:08.223583+0000","last_clean":"2023-10-09T06:34:08.223583+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:34:08.223583+0000","last_undegraded":"2023-10-09T06:34:08.223583+0000","last_fullsized":"2023-10-09T06:34:08.223583+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224078+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:34:08.224078+0000","last_peered":"2023-10-09T06:34:08.224078+0000","last_clean":"2023-10-09T06:34:08.224078+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:34:08.224078+0000","last_undegraded":"2023-10-09T06:34:08.224078+0000","last_fullsized":"2023-10-09T06:34:08.224078+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"198'5231","reported_seq":5246,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:57.449599+0000","last_change":"2023-10-09T06:33:33.319285+0000","last_active":"2023-10-09T06:34:57.449599+0000","last_peered":"2023-10-09T06:34:57.449599+0000","last_clean":"2023-10-09T06:34:57.449599+0000","last_became_active":"2023-10-09T06:33:33.318947+0000","last_became_peered":"2023-10-09T06:33:33.318947+0000","last_unstale":"2023-10-09T06:34:57.449599+0000","last_undegraded":"2023-10-09T06:34:57.449599+0000","last_fullsized":"2023-10-09T06:34:57.449599+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5231,"log_dups_size":0,"ondisk_log_size":5231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:22.467861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342818816,"num_objects":5231,"num_object_clones":0,"num_object_copies":15693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5231,"num_write_kb":334784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":194,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.521150+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:34:11.521150+0000","last_peered":"2023-10-09T06:34:11.521150+0000","last_clean":"2023-10-09T06:34:11.521150+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:34:11.521150+0000","last_undegraded":"2023-10-09T06:34:11.521150+0000","last_fullsized":"2023-10-09T06:34:11.521150+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":151,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213975+0000","last_change":"2023-10-09T06:14:42.684981+0000","last_active":"2023-10-09T06:34:08.213975+0000","last_peered":"2023-10-09T06:34:08.213975+0000","last_clean":"2023-10-09T06:34:08.213975+0000","last_became_active":"2023-10-09T06:14:42.684677+0000","last_became_peered":"2023-10-09T06:14:42.684677+0000","last_unstale":"2023-10-09T06:34:08.213975+0000","last_undegraded":"2023-10-09T06:34:08.213975+0000","last_fullsized":"2023-10-09T06:34:08.213975+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:43.680992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"198'5395","reported_seq":5410,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:58.752801+0000","last_change":"2023-10-09T06:33:33.317867+0000","last_active":"2023-10-09T06:34:58.752801+0000","last_peered":"2023-10-09T06:34:58.752801+0000","last_clean":"2023-10-09T06:34:58.752801+0000","last_became_active":"2023-10-09T06:33:33.317504+0000","last_became_peered":"2023-10-09T06:33:33.317504+0000","last_unstale":"2023-10-09T06:34:58.752801+0000","last_undegraded":"2023-10-09T06:34:58.752801+0000","last_fullsized":"2023-10-09T06:34:58.752801+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5395,"log_dups_size":0,"ondisk_log_size":5395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:56.906981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353566720,"num_objects":5395,"num_object_clones":0,"num_object_copies":16185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5395,"num_write_kb":345280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1052,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:32.457509+0000","last_change":"2023-10-09T06:22:46.885956+0000","last_active":"2023-10-09T06:34:32.457509+0000","last_peered":"2023-10-09T06:34:32.457509+0000","last_clean":"2023-10-09T06:34:32.457509+0000","last_became_active":"2023-10-09T06:22:46.885075+0000","last_became_peered":"2023-10-09T06:22:46.885075+0000","last_unstale":"2023-10-09T06:34:32.457509+0000","last_undegraded":"2023-10-09T06:34:32.457509+0000","last_fullsized":"2023-10-09T06:34:32.457509+0000","mapping_epoch":140,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:34.426374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225321+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:34:08.225321+0000","last_peered":"2023-10-09T06:34:08.225321+0000","last_clean":"2023-10-09T06:34:08.225321+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:34:08.225321+0000","last_undegraded":"2023-10-09T06:34:08.225321+0000","last_fullsized":"2023-10-09T06:34:08.225321+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":137,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:32.457431+0000","last_change":"2023-10-09T06:13:43.432977+0000","last_active":"2023-10-09T06:34:32.457431+0000","last_peered":"2023-10-09T06:34:32.457431+0000","last_clean":"2023-10-09T06:34:32.457431+0000","last_became_active":"2023-10-09T06:13:43.432657+0000","last_became_peered":"2023-10-09T06:13:43.432657+0000","last_unstale":"2023-10-09T06:34:32.457431+0000","last_undegraded":"2023-10-09T06:34:32.457431+0000","last_fullsized":"2023-10-09T06:34:32.457431+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:18.587888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"198'5332","reported_seq":5346,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.682549+0000","last_change":"2023-10-09T06:33:33.593915+0000","last_active":"2023-10-09T06:34:59.682549+0000","last_peered":"2023-10-09T06:34:59.682549+0000","last_clean":"2023-10-09T06:34:59.682549+0000","last_became_active":"2023-10-09T06:33:33.593144+0000","last_became_peered":"2023-10-09T06:33:33.593144+0000","last_unstale":"2023-10-09T06:34:59.682549+0000","last_undegraded":"2023-10-09T06:34:59.682549+0000","last_fullsized":"2023-10-09T06:34:59.682549+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5332,"num_write_kb":341248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":836,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225694+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:34:08.225694+0000","last_peered":"2023-10-09T06:34:08.225694+0000","last_clean":"2023-10-09T06:34:08.225694+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:34:08.225694+0000","last_undegraded":"2023-10-09T06:34:08.225694+0000","last_fullsized":"2023-10-09T06:34:08.225694+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":181,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224424+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:34:08.224424+0000","last_peered":"2023-10-09T06:34:08.224424+0000","last_clean":"2023-10-09T06:34:08.224424+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:34:08.224424+0000","last_undegraded":"2023-10-09T06:34:08.224424+0000","last_fullsized":"2023-10-09T06:34:08.224424+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":167,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:32.457471+0000","last_change":"2023-10-09T06:14:42.686071+0000","last_active":"2023-10-09T06:34:32.457471+0000","last_peered":"2023-10-09T06:34:32.457471+0000","last_clean":"2023-10-09T06:34:32.457471+0000","last_became_active":"2023-10-09T06:14:42.685698+0000","last_became_peered":"2023-10-09T06:14:42.685698+0000","last_unstale":"2023-10-09T06:34:32.457471+0000","last_undegraded":"2023-10-09T06:34:32.457471+0000","last_fullsized":"2023-10-09T06:34:32.457471+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:53.917364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"198'5226","reported_seq":5241,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:56.556195+0000","last_change":"2023-10-09T06:33:33.319622+0000","last_active":"2023-10-09T06:34:56.556195+0000","last_peered":"2023-10-09T06:34:56.556195+0000","last_clean":"2023-10-09T06:34:56.556195+0000","last_became_active":"2023-10-09T06:33:33.319140+0000","last_became_peered":"2023-10-09T06:33:33.319140+0000","last_unstale":"2023-10-09T06:34:56.556195+0000","last_undegraded":"2023-10-09T06:34:56.556195+0000","last_fullsized":"2023-10-09T06:34:56.556195+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:00.867299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342491136,"num_objects":5226,"num_object_clones":0,"num_object_copies":15678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5226,"num_write_kb":334464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":562,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.221948+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:34:08.221948+0000","last_peered":"2023-10-09T06:34:08.221948+0000","last_clean":"2023-10-09T06:34:08.221948+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:34:08.221948+0000","last_undegraded":"2023-10-09T06:34:08.221948+0000","last_fullsized":"2023-10-09T06:34:08.221948+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":169,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.212985+0000","last_change":"2023-10-09T06:28:31.018454+0000","last_active":"2023-10-09T06:34:08.212985+0000","last_peered":"2023-10-09T06:34:08.212985+0000","last_clean":"2023-10-09T06:34:08.212985+0000","last_became_active":"2023-10-09T06:28:31.018061+0000","last_became_peered":"2023-10-09T06:28:31.018061+0000","last_unstale":"2023-10-09T06:34:08.212985+0000","last_undegraded":"2023-10-09T06:34:08.212985+0000","last_fullsized":"2023-10-09T06:34:08.212985+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.185164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213940+0000","last_change":"2023-10-09T06:13:43.427445+0000","last_active":"2023-10-09T06:34:08.213940+0000","last_peered":"2023-10-09T06:34:08.213940+0000","last_clean":"2023-10-09T06:34:08.213940+0000","last_became_active":"2023-10-09T06:13:43.427081+0000","last_became_peered":"2023-10-09T06:13:43.427081+0000","last_unstale":"2023-10-09T06:34:08.213940+0000","last_undegraded":"2023-10-09T06:34:08.213940+0000","last_fullsized":"2023-10-09T06:34:08.213940+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:16.485757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"198'5462","reported_seq":5476,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.647647+0000","last_change":"2023-10-09T06:33:33.244731+0000","last_active":"2023-10-09T06:34:59.647647+0000","last_peered":"2023-10-09T06:34:59.647647+0000","last_clean":"2023-10-09T06:34:59.647647+0000","last_became_active":"2023-10-09T06:33:33.243726+0000","last_became_peered":"2023-10-09T06:33:33.243726+0000","last_unstale":"2023-10-09T06:34:59.647647+0000","last_undegraded":"2023-10-09T06:34:59.647647+0000","last_fullsized":"2023-10-09T06:34:59.647647+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5462,"log_dups_size":0,"ondisk_log_size":5462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357957632,"num_objects":5462,"num_object_clones":0,"num_object_copies":16386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5462,"num_write_kb":349568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":546,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225431+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:34:08.225431+0000","last_peered":"2023-10-09T06:34:08.225431+0000","last_clean":"2023-10-09T06:34:08.225431+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:34:08.225431+0000","last_undegraded":"2023-10-09T06:34:08.225431+0000","last_fullsized":"2023-10-09T06:34:08.225431+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":440,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224083+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:34:08.224083+0000","last_peered":"2023-10-09T06:34:08.224083+0000","last_clean":"2023-10-09T06:34:08.224083+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:34:08.224083+0000","last_undegraded":"2023-10-09T06:34:08.224083+0000","last_fullsized":"2023-10-09T06:34:08.224083+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":184,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.218883+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:34:08.218883+0000","last_peered":"2023-10-09T06:34:08.218883+0000","last_clean":"2023-10-09T06:34:08.218883+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:34:08.218883+0000","last_undegraded":"2023-10-09T06:34:08.218883+0000","last_fullsized":"2023-10-09T06:34:08.218883+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":153,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224395+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:34:08.224395+0000","last_peered":"2023-10-09T06:34:08.224395+0000","last_clean":"2023-10-09T06:34:08.224395+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:34:08.224395+0000","last_undegraded":"2023-10-09T06:34:08.224395+0000","last_fullsized":"2023-10-09T06:34:08.224395+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"198'5334","reported_seq":5348,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:57.453087+0000","last_change":"2023-10-09T06:33:33.319537+0000","last_active":"2023-10-09T06:34:57.453087+0000","last_peered":"2023-10-09T06:34:57.453087+0000","last_clean":"2023-10-09T06:34:57.453087+0000","last_became_active":"2023-10-09T06:33:33.319150+0000","last_became_peered":"2023-10-09T06:33:33.319150+0000","last_unstale":"2023-10-09T06:34:57.453087+0000","last_undegraded":"2023-10-09T06:34:57.453087+0000","last_fullsized":"2023-10-09T06:34:57.453087+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:18.214528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349569024,"num_objects":5334,"num_object_clones":0,"num_object_copies":16002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":341376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":119,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223944+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:34:08.223944+0000","last_peered":"2023-10-09T06:34:08.223944+0000","last_clean":"2023-10-09T06:34:08.223944+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:34:08.223944+0000","last_undegraded":"2023-10-09T06:34:08.223944+0000","last_fullsized":"2023-10-09T06:34:08.223944+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":155,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223592+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:34:08.223592+0000","last_peered":"2023-10-09T06:34:08.223592+0000","last_clean":"2023-10-09T06:34:08.223592+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:34:08.223592+0000","last_undegraded":"2023-10-09T06:34:08.223592+0000","last_fullsized":"2023-10-09T06:34:08.223592+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":170,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.935723+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:34:39.935723+0000","last_peered":"2023-10-09T06:34:39.935723+0000","last_clean":"2023-10-09T06:34:39.935723+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:34:39.935723+0000","last_undegraded":"2023-10-09T06:34:39.935723+0000","last_fullsized":"2023-10-09T06:34:39.935723+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"198'5228","reported_seq":5242,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:56.586574+0000","last_change":"2023-10-09T06:33:33.320380+0000","last_active":"2023-10-09T06:34:56.586574+0000","last_peered":"2023-10-09T06:34:56.586574+0000","last_clean":"2023-10-09T06:34:56.586574+0000","last_became_active":"2023-10-09T06:33:33.319960+0000","last_became_peered":"2023-10-09T06:33:33.319960+0000","last_unstale":"2023-10-09T06:34:56.586574+0000","last_undegraded":"2023-10-09T06:34:56.586574+0000","last_fullsized":"2023-10-09T06:34:56.586574+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5228,"log_dups_size":0,"ondisk_log_size":5228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:25.704605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342622208,"num_objects":5228,"num_object_clones":0,"num_object_copies":15684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5228,"num_write_kb":334592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":108,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224637+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:34:08.224637+0000","last_peered":"2023-10-09T06:34:08.224637+0000","last_clean":"2023-10-09T06:34:08.224637+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:34:08.224637+0000","last_undegraded":"2023-10-09T06:34:08.224637+0000","last_fullsized":"2023-10-09T06:34:08.224637+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:32.457491+0000","last_change":"2023-10-09T06:14:12.892439+0000","last_active":"2023-10-09T06:34:32.457491+0000","last_peered":"2023-10-09T06:34:32.457491+0000","last_clean":"2023-10-09T06:34:32.457491+0000","last_became_active":"2023-10-09T06:14:12.892065+0000","last_became_peered":"2023-10-09T06:14:12.892065+0000","last_unstale":"2023-10-09T06:34:32.457491+0000","last_undegraded":"2023-10-09T06:34:32.457491+0000","last_fullsized":"2023-10-09T06:34:32.457491+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:33.778698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":167,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215429+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:34:08.215429+0000","last_peered":"2023-10-09T06:34:08.215429+0000","last_clean":"2023-10-09T06:34:08.215429+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:34:08.215429+0000","last_undegraded":"2023-10-09T06:34:08.215429+0000","last_fullsized":"2023-10-09T06:34:08.215429+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"198'5562","reported_seq":5577,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:58.797925+0000","last_change":"2023-10-09T06:33:33.586554+0000","last_active":"2023-10-09T06:34:58.797925+0000","last_peered":"2023-10-09T06:34:58.797925+0000","last_clean":"2023-10-09T06:34:58.797925+0000","last_became_active":"2023-10-09T06:33:33.586066+0000","last_became_peered":"2023-10-09T06:33:33.586066+0000","last_unstale":"2023-10-09T06:34:58.797925+0000","last_undegraded":"2023-10-09T06:34:58.797925+0000","last_fullsized":"2023-10-09T06:34:58.797925+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.973752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364511232,"num_objects":5562,"num_object_clones":0,"num_object_copies":16686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5562,"num_write_kb":355968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":200,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.521267+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:34:11.521267+0000","last_peered":"2023-10-09T06:34:11.521267+0000","last_clean":"2023-10-09T06:34:11.521267+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:34:11.521267+0000","last_undegraded":"2023-10-09T06:34:11.521267+0000","last_fullsized":"2023-10-09T06:34:11.521267+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":581,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.213653+0000","last_change":"2023-10-09T06:28:34.035388+0000","last_active":"2023-10-09T06:34:08.213653+0000","last_peered":"2023-10-09T06:34:08.213653+0000","last_clean":"2023-10-09T06:34:08.213653+0000","last_became_active":"2023-10-09T06:28:34.034893+0000","last_became_peered":"2023-10-09T06:28:34.034893+0000","last_unstale":"2023-10-09T06:34:08.213653+0000","last_undegraded":"2023-10-09T06:34:08.213653+0000","last_fullsized":"2023-10-09T06:34:08.213653+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:43.453696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":199,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.934951+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:34:39.934951+0000","last_peered":"2023-10-09T06:34:39.934951+0000","last_clean":"2023-10-09T06:34:39.934951+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:34:39.934951+0000","last_undegraded":"2023-10-09T06:34:39.934951+0000","last_fullsized":"2023-10-09T06:34:39.934951+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"198'5375","reported_seq":5389,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:59.639431+0000","last_change":"2023-10-09T06:33:33.594495+0000","last_active":"2023-10-09T06:34:59.639431+0000","last_peered":"2023-10-09T06:34:59.639431+0000","last_clean":"2023-10-09T06:34:59.639431+0000","last_became_active":"2023-10-09T06:33:33.593206+0000","last_became_peered":"2023-10-09T06:33:33.593206+0000","last_unstale":"2023-10-09T06:34:59.639431+0000","last_undegraded":"2023-10-09T06:34:59.639431+0000","last_fullsized":"2023-10-09T06:34:59.639431+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352256000,"num_objects":5375,"num_object_clones":0,"num_object_copies":16125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5375,"num_write_kb":344000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":718,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.222185+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:34:08.222185+0000","last_peered":"2023-10-09T06:34:08.222185+0000","last_clean":"2023-10-09T06:34:08.222185+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:34:08.222185+0000","last_undegraded":"2023-10-09T06:34:08.222185+0000","last_fullsized":"2023-10-09T06:34:08.222185+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":159,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.935654+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:34:39.935654+0000","last_peered":"2023-10-09T06:34:39.935654+0000","last_clean":"2023-10-09T06:34:39.935654+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:34:39.935654+0000","last_undegraded":"2023-10-09T06:34:39.935654+0000","last_fullsized":"2023-10-09T06:34:39.935654+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223605+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:34:08.223605+0000","last_peered":"2023-10-09T06:34:08.223605+0000","last_clean":"2023-10-09T06:34:08.223605+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:34:08.223605+0000","last_undegraded":"2023-10-09T06:34:08.223605+0000","last_fullsized":"2023-10-09T06:34:08.223605+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215272+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:34:08.215272+0000","last_peered":"2023-10-09T06:34:08.215272+0000","last_clean":"2023-10-09T06:34:08.215272+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:34:08.215272+0000","last_undegraded":"2023-10-09T06:34:08.215272+0000","last_fullsized":"2023-10-09T06:34:08.215272+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":157,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:11.225748+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:34:11.225748+0000","last_peered":"2023-10-09T06:34:11.225748+0000","last_clean":"2023-10-09T06:34:11.225748+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:34:11.225748+0000","last_undegraded":"2023-10-09T06:34:11.225748+0000","last_fullsized":"2023-10-09T06:34:11.225748+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":165,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225705+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:34:08.225705+0000","last_peered":"2023-10-09T06:34:08.225705+0000","last_clean":"2023-10-09T06:34:08.225705+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:34:08.225705+0000","last_undegraded":"2023-10-09T06:34:08.225705+0000","last_fullsized":"2023-10-09T06:34:08.225705+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224575+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:34:08.224575+0000","last_peered":"2023-10-09T06:34:08.224575+0000","last_clean":"2023-10-09T06:34:08.224575+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:34:08.224575+0000","last_undegraded":"2023-10-09T06:34:08.224575+0000","last_fullsized":"2023-10-09T06:34:08.224575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":176,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.225355+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:34:08.225355+0000","last_peered":"2023-10-09T06:34:08.225355+0000","last_clean":"2023-10-09T06:34:08.225355+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:34:08.225355+0000","last_undegraded":"2023-10-09T06:34:08.225355+0000","last_fullsized":"2023-10-09T06:34:08.225355+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.224168+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:34:08.224168+0000","last_peered":"2023-10-09T06:34:08.224168+0000","last_clean":"2023-10-09T06:34:08.224168+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:34:08.224168+0000","last_undegraded":"2023-10-09T06:34:08.224168+0000","last_fullsized":"2023-10-09T06:34:08.224168+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":200,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-10-09T06:34:39.925782+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:34:39.925782+0000","last_peered":"2023-10-09T06:34:39.925782+0000","last_clean":"2023-10-09T06:34:39.925782+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:34:39.925782+0000","last_undegraded":"2023-10-09T06:34:39.925782+0000","last_fullsized":"2023-10-09T06:34:39.925782+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.218373+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:34:08.218373+0000","last_peered":"2023-10-09T06:34:08.218373+0000","last_clean":"2023-10-09T06:34:08.218373+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:34:08.218373+0000","last_undegraded":"2023-10-09T06:34:08.218373+0000","last_fullsized":"2023-10-09T06:34:08.218373+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":159,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.215633+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:34:08.215633+0000","last_peered":"2023-10-09T06:34:08.215633+0000","last_clean":"2023-10-09T06:34:08.215633+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:34:08.215633+0000","last_undegraded":"2023-10-09T06:34:08.215633+0000","last_fullsized":"2023-10-09T06:34:08.215633+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.223587+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:34:08.223587+0000","last_peered":"2023-10-09T06:34:08.223587+0000","last_clean":"2023-10-09T06:34:08.223587+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:34:08.223587+0000","last_undegraded":"2023-10-09T06:34:08.223587+0000","last_fullsized":"2023-10-09T06:34:08.223587+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5632557056,"num_objects":85946,"num_object_clones":0,"num_object_copies":257838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85946,"num_write_kb":5500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16902258688,"data_stored":16902258688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85946,"ondisk_log_size":85946,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430644,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037164,"kb_used_data":1731748,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91707988,"statfs":{"total":95995035648,"available":93908979712,"internally_reserved":0,"allocated":1773309952,"data_stored":1772752419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:56 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.3100000000000001,"15min":2.9079999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.6080000000000001,"5min":12.048,"15min":178.738},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.3979999999999999,"15min":2.8740000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.252},"max":{"1min":3.2519999999999998,"5min":15.465,"15min":177.75},"last":3.2519999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":7.8849999999999998,"5min":2.552,"15min":2.4940000000000002},"min":{"1min":0.38400000000000001,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":148.88499999999999,"5min":148.88499999999999,"15min":148.88499999999999},"last":2.8330000000000002},{"interface":"front","average":{"1min":7.7610000000000001,"5min":2.6909999999999998,"15min":2.3809999999999998},"min":{"1min":0.38200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":148.059,"5min":148.059,"15min":148.059},"last":0.60199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:34:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.2849999999999999,"15min":2.4100000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29399999999999998},"max":{"1min":2.8180000000000001,"5min":15.585000000000001,"15min":104.992},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.3160000000000001,"15min":2.2930000000000001},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.456,"5min":7.0629999999999997,"15min":104.691},"last":0.49199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.7890000000000001,"15min":3.573},"min":{"1min":0.42499999999999999,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":3.323,"5min":90.328000000000003,"15min":183.971},"last":1.4410000000000001},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.5590000000000002,"15min":3.3599999999999999},"min":{"1min":0.48099999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.5339999999999998,"5min":90.295000000000002,"15min":183.095},"last":1.333}]},{"osd":4,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.018,"15min":1.2370000000000001},"min":{"1min":0.57699999999999996,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.397,"5min":6.6239999999999997,"15min":11.029999999999999},"last":3.2080000000000002},{"interface":"front","average":{"1min":1.002,"5min":1.298,"15min":1.5980000000000001},"min":{"1min":0.54000000000000004,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.341,"5min":15.504,"15min":15.504},"last":0.625}]},{"osd":5,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.0449999999999999,"15min":1.3999999999999999},"min":{"1min":0.626,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.4450000000000001,"5min":7.1550000000000002,"15min":11.996},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.012,"15min":1.3939999999999999},"min":{"1min":0.64200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.631,"5min":5.4909999999999997,"15min":13.634},"last":6.8940000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:33:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.4259999999999999,"15min":1.3260000000000001},"min":{"1min":0.64900000000000002,"5min":0.314,"15min":0.314},"max":{"1min":1.6879999999999999,"5min":15.521000000000001,"15min":15.521000000000001},"last":3.2879999999999998},{"interface":"front","average":{"1min":1.115,"5min":1.347,"15min":1.5049999999999999},"min":{"1min":0.496,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":1.5109999999999999,"5min":15.449999999999999,"15min":15.449999999999999},"last":0.65200000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626876,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2032156,"kb_used_data":1734932,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91712996,"statfs":{"total":95995035648,"available":93914107904,"internally_reserved":0,"allocated":1776570368,"data_stored":1776028462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.26,"15min":1.242},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.9119999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":3.2210000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.2709999999999999,"15min":1.3360000000000001},"min":{"1min":0.30499999999999999,"5min":0.29799999999999999,"15min":0.26300000000000001},"max":{"1min":3.0590000000000002,"5min":6.4240000000000004,"15min":8.2639999999999993},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":13.691000000000001,"5min":3.875,"15min":2.673},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":316.88099999999997,"5min":316.88099999999997,"15min":316.88099999999997},"last":2.7989999999999999},{"interface":"front","average":{"1min":13.871,"5min":4.0250000000000004,"15min":2.7189999999999999},"min":{"1min":0.39900000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":316.84399999999999,"5min":316.84399999999999,"15min":316.84399999999999},"last":0.29799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":2.355,"15min":2.4039999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":3.9580000000000002,"5min":53.616,"15min":95.540000000000006},"last":2.823},{"interface":"front","average":{"1min":0.96599999999999997,"5min":2.4340000000000002,"15min":2.371},"min":{"1min":0.35599999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":2.4060000000000001,"5min":53.665999999999997,"15min":96.352000000000004},"last":3.1440000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.23,"15min":2.3599999999999999},"min":{"1min":0.77300000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":2.468,"5min":27.491,"15min":115.09399999999999},"last":5.3979999999999997},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.8380000000000001,"15min":2.0089999999999999},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":1.3009999999999999,"5min":26.815000000000001,"15min":112.886},"last":2.6269999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.98399999999999999,"15min":1.2569999999999999},"min":{"1min":0.39200000000000002,"5min":0.32500000000000001,"15min":0.26000000000000001},"max":{"1min":2.6059999999999999,"5min":6.3799999999999999,"15min":7.3090000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.0529999999999999,"15min":1.419},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":4.0890000000000004,"5min":5.1609999999999996,"15min":10.300000000000001},"last":0.39500000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":0.98399999999999999,"15min":1.4550000000000001},"min":{"1min":0.42799999999999999,"5min":0.35499999999999998,"15min":0.34300000000000003},"max":{"1min":4.9619999999999997,"5min":6.4850000000000003,"15min":11.754},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.1699999999999999,"15min":1.589},"min":{"1min":0.41899999999999998,"5min":0.38200000000000001,"15min":0.308},"max":{"1min":6.423,"5min":6.423,"15min":12.295999999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.083,"15min":1.502},"min":{"1min":0.81299999999999994,"5min":0.38100000000000001,"15min":0.34000000000000002},"max":{"1min":1.651,"5min":6.5110000000000001,"15min":11.076000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.27,"5min":1.127,"15min":1.4330000000000001},"min":{"1min":0.94999999999999996,"5min":0.38500000000000001,"15min":0.38},"max":{"1min":2.52,"5min":9.4179999999999993,"15min":11.102},"last":0.65500000000000003}]}]},{"osd":1,"up_from":93,"seq":399431958785,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2747680,"kb_used_data":2366872,"kb_used_omap":7,"kb_used_meta":380792,"kb_avail":90997472,"statfs":{"total":95995035648,"available":93181411328,"internally_reserved":0,"allocated":2423676928,"data_stored":2422840713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":389931408},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,1,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":4.0869999999999997,"15min":2.5680000000000001},"min":{"1min":0.33600000000000002,"5min":0.187,"15min":0.187},"max":{"1min":6.4960000000000004,"5min":132.886,"15min":132.886},"last":3.516},{"interface":"front","average":{"1min":1.321,"5min":4.0970000000000004,"15min":2.7650000000000001},"min":{"1min":0.33800000000000002,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":9.7409999999999997,"5min":133.315,"15min":133.315},"last":0.93700000000000006}]},{"osd":2,"last update":"Mon Oct 9 06:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.6440000000000001,"15min":1.8740000000000001},"min":{"1min":0.35499999999999998,"5min":0.214,"15min":0.19800000000000001},"max":{"1min":1.569,"5min":67.206000000000003,"15min":67.206000000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.71899999999999997,"5min":2.923,"15min":1.948},"min":{"1min":0.309,"5min":0.224,"15min":0.224},"max":{"1min":1.099,"5min":67.132000000000005,"15min":67.132000000000005},"last":1.159}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":5.0490000000000004,"15min":4.0359999999999996},"min":{"1min":0.44700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":4.4290000000000003,"5min":132.792,"15min":132.792},"last":3.6859999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":4.9589999999999996,"15min":3.9980000000000002},"min":{"1min":0.40300000000000002,"5min":0.24099999999999999,"15min":0.23200000000000001},"max":{"1min":3.1589999999999998,"5min":132.53299999999999,"15min":132.53299999999999},"last":8.7780000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.5800000000000001,"15min":1.5149999999999999},"min":{"1min":0.66200000000000003,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":4.3360000000000003,"5min":26.068999999999999,"15min":26.068999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.034,"5min":1.391,"15min":1.5509999999999999},"min":{"1min":0.39100000000000001,"5min":0.311,"15min":0.31},"max":{"1min":2.7000000000000002,"5min":5.6689999999999996,"15min":12.907999999999999},"last":0.72099999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.6299999999999999,"15min":1.6180000000000001},"min":{"1min":0.47199999999999998,"5min":0.35899999999999999,"15min":0.32500000000000001},"max":{"1min":10.218,"5min":10.586,"15min":14.170999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.462,"5min":1.5600000000000001,"15min":1.6559999999999999},"min":{"1min":0.65600000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.0279999999999996,"5min":26.157,"15min":26.157},"last":0.88700000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.6399999999999999,"15min":1.538},"min":{"1min":0.502,"5min":0.38600000000000001,"15min":0.36699999999999999},"max":{"1min":10.19,"5min":10.625,"15min":14.212999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.663,"15min":1.6559999999999999},"min":{"1min":0.64400000000000002,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":4.3799999999999999,"5min":26.132000000000001,"15min":26.132000000000001},"last":0.80900000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.9059999999999999,"15min":1.988},"min":{"1min":0.63900000000000001,"5min":0.41099999999999998,"15min":0.35399999999999998},"max":{"1min":4.3620000000000001,"5min":26.109000000000002,"15min":30.774999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.111,"5min":1.486,"15min":1.6799999999999999},"min":{"1min":0.439,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":2.7229999999999999,"5min":5.8289999999999997,"15min":30.815000000000001},"last":0.68899999999999995}]}]},{"osd":0,"up_from":88,"seq":377957122311,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2566212,"kb_used_data":2093372,"kb_used_omap":8,"kb_used_meta":472823,"kb_avail":91178940,"statfs":{"total":95995035648,"available":93367234560,"internally_reserved":0,"allocated":2143612928,"data_stored":2142801604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":484171008},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":2.7629999999999999,"15min":2.5880000000000001},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4139999999999997,"5min":13.598000000000001,"15min":138.31700000000001},"last":5.8659999999999997},{"interface":"front","average":{"1min":0.95599999999999996,"5min":2.2400000000000002,"15min":2.355},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20200000000000001},"max":{"1min":3.8879999999999999,"5min":12.101000000000001,"15min":144.37},"last":5.6580000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":2.6429999999999998,"15min":1.972},"min":{"1min":0.35299999999999998,"5min":0.185,"15min":0.185},"max":{"1min":9.8079999999999998,"5min":13.425000000000001,"15min":28.745000000000001},"last":5.9029999999999996},{"interface":"front","average":{"1min":1.2609999999999999,"5min":2.0209999999999999,"15min":1.839},"min":{"1min":0.24299999999999999,"5min":0.22600000000000001,"15min":0.18099999999999999},"max":{"1min":3.9289999999999998,"5min":16.129000000000001,"15min":28.146999999999998},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.5129999999999999,"15min":2.3639999999999999},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.20899999999999999},"max":{"1min":6.4000000000000004,"5min":14.628,"15min":59.643000000000001},"last":5.1070000000000002},{"interface":"front","average":{"1min":1.119,"5min":2.3929999999999998,"15min":2.2149999999999999},"min":{"1min":0.43099999999999999,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.956,"5min":12.791,"15min":59.762999999999998},"last":2.1579999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.611,"15min":1.5509999999999999},"min":{"1min":0.38500000000000001,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":1.2849999999999999,"5min":8.9920000000000009,"15min":9.0890000000000004},"last":5.157},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.966,"15min":1.8360000000000001},"min":{"1min":0.54400000000000004,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.9710000000000001,"5min":9.2319999999999993,"15min":28.629000000000001},"last":5.9320000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":2.0739999999999998,"15min":2.0630000000000002},"min":{"1min":0.70299999999999996,"5min":0.36199999999999999,"15min":0.35399999999999998},"max":{"1min":3.0059999999999998,"5min":12.356999999999999,"15min":28.574000000000002},"last":6.8410000000000002},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.873,"15min":1.8089999999999999},"min":{"1min":0.54000000000000004,"5min":0.36299999999999999,"15min":0.32600000000000001},"max":{"1min":1.613,"5min":9.8209999999999997,"15min":28.061},"last":3.915}]},{"osd":6,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.8360000000000001,"15min":1.738},"min":{"1min":0.59499999999999997,"5min":0.35299999999999998,"15min":0.34599999999999997},"max":{"1min":1.548,"5min":16.181999999999999,"15min":27.966999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.6120000000000001,"15min":1.4710000000000001},"min":{"1min":0.58399999999999996,"5min":0.39300000000000002,"15min":0.33700000000000002},"max":{"1min":1.3300000000000001,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":5.1859999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.8029999999999999,"15min":1.833},"min":{"1min":0.63200000000000001,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":1.6599999999999999,"5min":7.9169999999999998,"15min":28.109999999999999},"last":3.871},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.8380000000000001,"15min":1.863},"min":{"1min":0.41799999999999998,"5min":0.41299999999999998,"15min":0.38400000000000001},"max":{"1min":1.5209999999999999,"5min":8.9190000000000005,"15min":28.074999999999999},"last":5.266}]}]},{"osd":2,"up_from":98,"seq":420906795259,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2438980,"kb_used_data":2004476,"kb_used_omap":11,"kb_used_meta":434484,"kb_avail":91306172,"statfs":{"total":95995035648,"available":93497520128,"internally_reserved":0,"allocated":2052583424,"data_stored":2051769392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":444912574},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":3.0790000000000002,"15min":2.2749999999999999},"min":{"1min":0.35199999999999998,"5min":0.189,"15min":0.17899999999999999},"max":{"1min":0.85899999999999999,"5min":117.2,"15min":117.2},"last":4.6950000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":2.6869999999999998,"15min":2.351},"min":{"1min":0.28499999999999998,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":0.76300000000000001,"5min":121.521,"15min":121.521},"last":2.2719999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4569999999999999,"5min":2.133,"15min":1.8109999999999999},"min":{"1min":0.41899999999999998,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":60.155000000000001,"5min":60.155000000000001,"15min":60.155000000000001},"last":4.7290000000000001},{"interface":"front","average":{"1min":3.5640000000000001,"5min":1.982,"15min":2.0219999999999998},"min":{"1min":0.35799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":62.582999999999998,"5min":62.582999999999998,"15min":62.582999999999998},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":3.1800000000000002,"15min":3.3250000000000002},"min":{"1min":0.253,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":2.1110000000000002,"5min":118.431,"15min":201.33600000000001},"last":5.3209999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":2.7799999999999998,"15min":3.2919999999999998},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.21199999999999999},"max":{"1min":2.113,"5min":118.64100000000001,"15min":199.95099999999999},"last":4.4630000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.359,"15min":1.9850000000000001},"min":{"1min":0.624,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.093,"5min":7.5759999999999996,"15min":19.184000000000001},"last":2.613},{"interface":"front","average":{"1min":0.872,"5min":1.2070000000000001,"15min":1.7889999999999999},"min":{"1min":0.628,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":1.3360000000000001,"5min":6.3049999999999997,"15min":29.408000000000001},"last":4.5759999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":2.335,"15min":2.3370000000000002},"min":{"1min":0.59399999999999997,"5min":0.35299999999999998,"15min":0.31},"max":{"1min":1.202,"5min":86.423000000000002,"15min":86.423000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":2.2400000000000002,"15min":2.0179999999999998},"min":{"1min":0.58599999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.2829999999999999,"5min":86.382999999999996,"15min":86.382999999999996},"last":4.5069999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.4910000000000001,"15min":1.6419999999999999},"min":{"1min":0.70699999999999996,"5min":0.38,"15min":0.32800000000000001},"max":{"1min":1.377,"5min":8.2959999999999994,"15min":29.504000000000001},"last":4.5469999999999997},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.4570000000000001,"15min":1.647},"min":{"1min":0.65200000000000002,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":1.1890000000000001,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":8.0869999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.595,"15min":1.772},"min":{"1min":0.69599999999999995,"5min":0.41599999999999998,"15min":0.308},"max":{"1min":1.2849999999999999,"5min":9.2050000000000001,"15min":9.2050000000000001},"last":4.7549999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.5600000000000001,"15min":2.0059999999999998},"min":{"1min":0.71399999999999997,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":1.2190000000000001,"5min":7.6920000000000002,"15min":12.051},"last":0.53500000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631733,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2481804,"kb_used_data":2011140,"kb_used_omap":3,"kb_used_meta":470652,"kb_avail":91263348,"statfs":{"total":95995035648,"available":93453668352,"internally_reserved":0,"allocated":2059407360,"data_stored":2058586028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":481948193},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.3769999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17399999999999999},"max":{"1min":3.23,"5min":15.285,"15min":147.988},"last":2.048},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.2050000000000001,"15min":2.4329999999999998},"min":{"1min":0.19500000000000001,"5min":0.19,"15min":0.18099999999999999},"max":{"1min":4.9589999999999996,"5min":23.646000000000001,"15min":147.94200000000001},"last":0.376}]},{"osd":1,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":2.5409999999999999,"15min":1.726},"min":{"1min":0.27400000000000002,"5min":0.19,"15min":0.19},"max":{"1min":7.8209999999999997,"5min":47.057000000000002,"15min":47.057000000000002},"last":3.2930000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.8839999999999999,"15min":2.0430000000000001},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.218},"max":{"1min":5.7249999999999996,"5min":54.752000000000002,"15min":54.752000000000002},"last":2.077}]},{"osd":2,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":2.9649999999999999,"15min":2.496},"min":{"1min":0.39900000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.1669999999999998,"5min":47.097999999999999,"15min":70.594999999999999},"last":2.0009999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":2.452,"15min":2.0329999999999999},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":3.0430000000000001,"5min":46.997,"15min":62.991999999999997},"last":3.3220000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.6509999999999998,"15min":1.978},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.411,"5min":72.653000000000006,"15min":72.653000000000006},"last":1.861},{"interface":"front","average":{"1min":0.80400000000000005,"5min":2.4430000000000001,"15min":2.0409999999999999},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":1.752,"5min":72.706999999999994,"15min":72.706999999999994},"last":3.1920000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.845,"15min":1.734},"min":{"1min":0.42699999999999999,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":2.3820000000000001,"5min":11.781000000000001,"15min":11.781000000000001},"last":3.234},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.9810000000000001,"15min":1.8380000000000001},"min":{"1min":0.48699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.363,"5min":9.4619999999999997,"15min":11.754},"last":1.927}]},{"osd":6,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.97,"15min":1.708},"min":{"1min":0.48399999999999999,"5min":0.38900000000000001,"15min":0.36099999999999999},"max":{"1min":3.0880000000000001,"5min":15.122999999999999,"15min":15.122999999999999},"last":2.0249999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.9790000000000001,"15min":1.653},"min":{"1min":0.38100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.964,"5min":11.728999999999999,"15min":11.728999999999999},"last":3.2639999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":2.0699999999999998,"15min":1.837},"min":{"1min":0.51900000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.3069999999999999,"5min":8.6059999999999999,"15min":18.695},"last":1.8939999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":2.0880000000000001,"15min":1.9419999999999999},"min":{"1min":0.47099999999999997,"5min":0.43099999999999999,"15min":0.39900000000000002},"max":{"1min":3.1429999999999998,"5min":13.211,"15min":13.211},"last":3.6309999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019342,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037412,"kb_used_data":1731996,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91707740,"statfs":{"total":95995035648,"available":93908725760,"internally_reserved":0,"allocated":1773563904,"data_stored":1773013949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.6259999999999999,"15min":2.3220000000000001},"min":{"1min":0.27100000000000002,"5min":0.254,"15min":0.254},"max":{"1min":4.1900000000000004,"5min":16.218,"15min":115.59699999999999},"last":0.374},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.738,"15min":2.6429999999999998},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.27700000000000002},"max":{"1min":5.8159999999999998,"5min":15.242000000000001,"15min":115.65000000000001},"last":3.835}]},{"osd":1,"last update":"Mon Oct 9 06:34:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":1.47,"15min":2.2639999999999998},"min":{"1min":0.45100000000000001,"5min":0.29199999999999998,"15min":0.27600000000000002},"max":{"1min":17.033000000000001,"5min":17.033000000000001,"15min":179.47300000000001},"last":0.877},{"interface":"front","average":{"1min":2.1360000000000001,"5min":1.6859999999999999,"15min":2.6699999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.28499999999999998},"max":{"1min":17.015999999999998,"5min":17.015999999999998,"15min":179.447},"last":0.93799999999999994}]},{"osd":2,"last update":"Mon Oct 9 06:34:58 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.6519999999999999,"15min":1.964},"min":{"1min":0.31,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":5.9020000000000001,"5min":14.705,"15min":61.783999999999999},"last":3.8799999999999999},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.659,"15min":2.0870000000000002},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":3.2789999999999999,"5min":9.7759999999999998,"15min":61.551000000000002},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:34:18 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":2.2250000000000001,"15min":2.3460000000000001},"min":{"1min":0.40300000000000002,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":7.2530000000000001,"5min":12.081,"15min":91.361999999999995},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.9950000000000001,"15min":1.9319999999999999},"min":{"1min":0.47999999999999998,"5min":0.36699999999999999,"15min":0.313},"max":{"1min":7.1870000000000003,"5min":16.257000000000001,"15min":89.676000000000002},"last":0.40999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:33 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.90000000000000002,"15min":1.2949999999999999},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.4550000000000001,"5min":11.737,"15min":12.242000000000001},"last":3.3290000000000002},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.234,"15min":1.708},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.26500000000000001},"max":{"1min":7.7649999999999997,"5min":7.7649999999999997,"15min":20.113},"last":3.9180000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.288,"15min":1.5820000000000001},"min":{"1min":0.54700000000000004,"5min":0.35399999999999998,"15min":0.29499999999999998},"max":{"1min":1.609,"5min":8.1110000000000007,"15min":18.010000000000002},"last":3.944},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.2809999999999999,"15min":1.4319999999999999},"min":{"1min":0.39100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4199999999999999,"5min":11.782,"15min":13.913},"last":3.3610000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.4419999999999999,"15min":1.776},"min":{"1min":0.73399999999999999,"5min":0.42399999999999999,"15min":0.371},"max":{"1min":2.6480000000000001,"5min":12.779,"15min":15.294},"last":4.4409999999999998},{"interface":"front","average":{"1min":1.006,"5min":1.226,"15min":1.7470000000000001},"min":{"1min":0.432,"5min":0.317,"15min":0.317},"max":{"1min":1.6399999999999999,"5min":9.1370000000000005,"15min":14.896000000000001},"last":0.59799999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823109,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3082320,"kb_used_data":2768712,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":90662832,"statfs":{"total":95995035648,"available":92838739968,"internally_reserved":0,"allocated":2835161088,"data_stored":2834626746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.29,"15min":2.6659999999999999},"min":{"1min":0.25600000000000001,"5min":0.255,"15min":0.247},"max":{"1min":4.9900000000000002,"5min":83.197000000000003,"15min":83.197000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.343,"5min":2.238,"15min":2.3929999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.1669999999999998,"5min":83.563999999999993,"15min":83.563999999999993},"last":0.54600000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:34:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.3939999999999999,"15min":2.4329999999999998},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":3.694,"5min":5.9299999999999997,"15min":161.71199999999999},"last":1.246},{"interface":"front","average":{"1min":1.98,"5min":1.698,"15min":2.7080000000000002},"min":{"1min":0.624,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":11.753,"5min":11.753,"15min":162.50200000000001},"last":1.355}]},{"osd":2,"last update":"Mon Oct 9 06:34:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.0459999999999998,"15min":2.2759999999999998},"min":{"1min":0.34200000000000003,"5min":0.313,"15min":0.28199999999999997},"max":{"1min":5.2039999999999997,"5min":54.981999999999999,"15min":54.981999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.1719999999999999,"5min":2.1499999999999999,"15min":2.173},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30499999999999999},"max":{"1min":3.7250000000000001,"5min":55.008000000000003,"15min":59.082999999999998},"last":0.65700000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":3.3839999999999999,"15min":2.8769999999999998},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":5.0670000000000002,"5min":84.174000000000007,"15min":84.174000000000007},"last":1.611},{"interface":"front","average":{"1min":1.153,"5min":3.4750000000000001,"15min":2.7210000000000001},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.786,"5min":83.114999999999995,"15min":83.114999999999995},"last":0.622}]},{"osd":4,"last update":"Mon Oct 9 06:34:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.288,"15min":1.8740000000000001},"min":{"1min":0.375,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.0990000000000002,"5min":8.8879999999999999,"15min":17.667999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.222,"15min":1.6080000000000001},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.308},"max":{"1min":5.0919999999999996,"5min":5.3040000000000003,"15min":16.181999999999999},"last":0.64100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.3320000000000001,"15min":1.7270000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.133,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.292,"5min":1.292,"15min":1.631},"min":{"1min":0.45500000000000002,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":5.1180000000000003,"5min":9.3450000000000006,"15min":12.365},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":1.2050000000000001,"15min":1.6220000000000001},"min":{"1min":0.53800000000000003,"5min":0.372,"15min":0.36899999999999999},"max":{"1min":5.7190000000000003,"5min":5.7190000000000003,"15min":12.711},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.25,"15min":1.4299999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.36799999999999999},"max":{"1min":3.7549999999999999,"5min":5.5919999999999996,"15min":11.981},"last":0.73699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2141585408,"data_stored":2141585408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2438201344,"data_stored":2438201344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2060713984,"data_stored":2060713984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2072838144,"data_stored":2072838144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1786314752,"data_stored":1786314752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2847080448,"data_stored":2847080448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1774452736,"data_stored":1774452736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1781071872,"data_stored":1781071872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:35:05.741 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:35:05.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-09T06:35:05.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 92113 92097 83.3893 60.3125 0.0179783 0.0119865 2023-10-09T06:35:05.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:05.844 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:05.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:05.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:05.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:06.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:06.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:06.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:06.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:06.216 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:06 smithi155 bash[59495]: cluster 2023-10-09T06:35:05.139593+0000 mgr.y (mgr.25723) 911 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T06:35:06.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:06.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:06.349 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:06 smithi125 bash[77299]: cluster 2023-10-09T06:35:05.139593+0000 mgr.y (mgr.25723) 911 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T06:35:06.350 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:06 smithi125 bash[80002]: cluster 2023-10-09T06:35:05.139593+0000 mgr.y (mgr.25723) 911 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-09T06:35:06.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:06.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:06.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:06.575 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:06.575 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:06.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:06.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:06.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:06.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:06.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 92999 92983 82.9891 55.375 0.0143202 0.0120436 2023-10-09T06:35:06.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:06.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:06.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:07.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:07.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:07.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:07.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:07.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:07.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:07.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:07.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:07.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:07.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:07.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:07.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:07.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:07.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:35:07.753 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:35:07.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:35:07.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 93905 93889 82.6178 56.625 0.035114 0.0120987 2023-10-09T06:35:07.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:07.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:07.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:07.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:08.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:08.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:08.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:08.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:08.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:08.291 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:08 smithi125 bash[77299]: cluster 2023-10-09T06:35:07.140165+0000 mgr.y (mgr.25723) 912 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T06:35:08.292 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:08 smithi125 bash[80002]: cluster 2023-10-09T06:35:07.140165+0000 mgr.y (mgr.25723) 912 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T06:35:08.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:08.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:08.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:08 smithi155 bash[59495]: cluster 2023-10-09T06:35:07.140165+0000 mgr.y (mgr.25723) 912 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T06:35:08.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:08.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:08.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:08.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:08.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:08.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 94756 94740 82.2091 53.1875 0.00794458 0.0121587 2023-10-09T06:35:08.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:08.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:08.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:08.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:09.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:09.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:09.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:09.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:09.284 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:09.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:09.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:09.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:09.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:09.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:09.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:09.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:09.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:09.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 95644 95628 81.8432 55.5 0.0355545 0.012213 2023-10-09T06:35:09.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:09.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:09.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:09.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:09.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:10.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:10.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:10.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:09 smithi155 bash[59495]: cluster 2023-10-09T06:35:09.140770+0000 mgr.y (mgr.25723) 913 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:35:10.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:10.200 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:09 smithi125 bash[80002]: cluster 2023-10-09T06:35:09.140770+0000 mgr.y (mgr.25723) 913 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:35:10.201 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:09 smithi125 bash[77299]: cluster 2023-10-09T06:35:09.140770+0000 mgr.y (mgr.25723) 913 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:35:10.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:10.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:10.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:10.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:10.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:10.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:10.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:10.630 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:10.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:10.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:10.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 96528 96512 81.4839 55.25 0.0310406 0.0122668 2023-10-09T06:35:10.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:10.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:10.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:10.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:10.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:11.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:11.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:11.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:11.159 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:11.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:11.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:11.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:11.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:11] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T06:35:11.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:11.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:11.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:11.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:11.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:11.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:11.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:11.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:11.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:11.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 97402 97386 81.1258 54.625 0.0168476 0.0123205 2023-10-09T06:35:11.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:11.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:11.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:11.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:11.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:12.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:12.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:12.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:12.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:12.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:12.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:12 smithi155 bash[59495]: cluster 2023-10-09T06:35:11.141360+0000 mgr.y (mgr.25723) 914 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:35:12.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:12.507 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:12.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:12.624 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:12.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:12 smithi125 bash[77299]: cluster 2023-10-09T06:35:11.141360+0000 mgr.y (mgr.25723) 914 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:35:12.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:12 smithi125 bash[80002]: cluster 2023-10-09T06:35:11.141360+0000 mgr.y (mgr.25723) 914 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:35:12.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:12.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:12.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:12.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 98302 98286 80.7985 56.25 0.00961381 0.0123711 2023-10-09T06:35:12.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:12.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:12.979 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:12.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:13.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:13.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:13.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:13.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:13.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:13.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:13.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:13.679 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:13.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.807 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 99120 99104 80.4131 51.125 0.00464257 0.0124192 2023-10-09T06:35:13.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:13.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:13.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:13.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:13.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:14.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:14.074 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:14.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:14.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:14.210 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:14 smithi125 bash[77299]: cluster 2023-10-09T06:35:13.142046+0000 mgr.y (mgr.25723) 915 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:35:14.311 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:14.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:14.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:14.478 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:14 smithi125 bash[80002]: cluster 2023-10-09T06:35:13.142046+0000 mgr.y (mgr.25723) 915 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:35:14.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:14.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:14 smithi155 bash[59495]: cluster 2023-10-09T06:35:13.142046+0000 mgr.y (mgr.25723) 915 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:35:14.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:14.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:14.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 99870 99854 79.9832 46.875 0.021072 0.0124964 2023-10-09T06:35:14.849 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:14.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:14.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:15.004 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:15.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:15.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:15.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:15.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:15.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:15.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:15.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:15.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:15.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:15.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:15.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 100594 100578 79.5436 45.25 0.0935091 0.0125573 2023-10-09T06:35:15.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:15.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:15.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:16.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:16.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:16.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:16.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:16.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:16.306 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:16 smithi155 bash[59495]: cluster 2023-10-09T06:35:15.143955+0000 mgr.y (mgr.25723) 916 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:35:16.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:16.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:16.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:16 smithi125 bash[77299]: cluster 2023-10-09T06:35:15.143955+0000 mgr.y (mgr.25723) 916 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:35:16.456 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:16 smithi125 bash[80002]: cluster 2023-10-09T06:35:15.143955+0000 mgr.y (mgr.25723) 916 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:35:16.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:16.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:16.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:16.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:16.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:35:16.801967+0000 min lat: 0.00131043 max lat: 0.352992 avg lat: 0.0126329 2023-10-09T06:35:16.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:35:16.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 101335 101319 79.1283 46.3125 0.0168998 0.0126329 2023-10-09T06:35:16.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:16.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:16.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:17.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:17.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:17.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:17.429 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:17.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:17.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.550 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:17.550 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:17.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:17.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:17.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 102186 102170 78.808 53.1875 0.00732623 0.0126818 2023-10-09T06:35:17.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:17.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:17.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:17.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:17.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:18.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:18.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:18.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:18.256 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:18.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:18.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:18.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:18.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:18 smithi155 bash[59495]: cluster 2023-10-09T06:35:17.144504+0000 mgr.y (mgr.25723) 917 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T06:35:18.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:18.503 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:18 smithi125 bash[77299]: cluster 2023-10-09T06:35:17.144504+0000 mgr.y (mgr.25723) 917 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T06:35:18.504 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:18 smithi125 bash[80002]: cluster 2023-10-09T06:35:17.144504+0000 mgr.y (mgr.25723) 917 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-09T06:35:18.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:18.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:18.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:35:18.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:18.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:18.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:18.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:18.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 103051 103035 78.5063 54.0625 0.0110623 0.0127328 2023-10-09T06:35:18.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:18.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:18.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:18.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:19.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:19.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:19.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:19 smithi125 bash[77299]: audit 2023-10-09T06:35:18.590027+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:19.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:19 smithi125 bash[80002]: audit 2023-10-09T06:35:18.590027+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:19.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:19.407 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:19.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:19.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:19.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:19 smithi155 bash[59495]: audit 2023-10-09T06:35:18.590027+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:19.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:19.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:19.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:19.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 103730 103714 78.0717 42.4375 0.00432623 0.0127712 2023-10-09T06:35:19.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:19.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:19.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:19.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:20.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:20.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:20.220 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:20 smithi155 bash[59495]: cluster 2023-10-09T06:35:19.145107+0000 mgr.y (mgr.25723) 918 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:35:20.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:20.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:20.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:20.362 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:20 smithi125 bash[77299]: cluster 2023-10-09T06:35:19.145107+0000 mgr.y (mgr.25723) 918 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:35:20.362 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:20 smithi125 bash[80002]: cluster 2023-10-09T06:35:19.145107+0000 mgr.y (mgr.25723) 918 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:35:20.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:20.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:20.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:20.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:20.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:20.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:20.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 104595 104579 77.7859 54.0625 0.0264778 0.0128503 2023-10-09T06:35:20.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:20.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:21.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:21.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:21.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:21.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:21.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:21.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:21.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:21.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:21.519 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:21] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:35:21.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:21.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:21.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:21.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:21.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:21.808 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 105336 105320 77.4156 46.3125 0.00472519 0.0129075 2023-10-09T06:35:21.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:21.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:21.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:21.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:21.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:22.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:22.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:22.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:22.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:22.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:22.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:22 smithi125 bash[77299]: cluster 2023-10-09T06:35:21.146002+0000 mgr.y (mgr.25723) 919 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:35:22.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:22 smithi125 bash[80002]: cluster 2023-10-09T06:35:21.146002+0000 mgr.y (mgr.25723) 919 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:35:22.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:22.444 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:22.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:22.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:22.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:22 smithi155 bash[59495]: cluster 2023-10-09T06:35:21.146002+0000 mgr.y (mgr.25723) 919 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-10-09T06:35:22.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:22.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:22.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 106141 106125 77.1005 50.3125 0.008016 0.0129633 2023-10-09T06:35:22.822 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:22.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:22.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:22.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:23.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:23.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:35:23.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:23.100 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:35:23.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:23.136 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:35:23.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:23.168 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:35:23.213 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:35:23.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:23.266 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:35:23.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:23.327 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:35:23.366 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:35:23.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:23 smithi155 bash[59495]: audit 2023-10-09T06:35:22.996066+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.125:0/2551445852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:23 smithi155 bash[59495]: audit 2023-10-09T06:35:22.998160+0000 mon.a (mon.0) 1115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:23 smithi155 bash[59495]: audit 2023-10-09T06:35:23.058117+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.125:0/1675059636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:23.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:23.404 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.416 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:35:23.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:23.483 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:23.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:23.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:23.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[77299]: audit 2023-10-09T06:35:22.996066+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.125:0/2551445852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[77299]: audit 2023-10-09T06:35:22.998160+0000 mon.a (mon.0) 1115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[77299]: audit 2023-10-09T06:35:23.058117+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.125:0/1675059636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:23.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[80002]: audit 2023-10-09T06:35:22.996066+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.125:0/2551445852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[80002]: audit 2023-10-09T06:35:22.998160+0000 mon.a (mon.0) 1115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:35:23.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:23 smithi125 bash[80002]: audit 2023-10-09T06:35:23.058117+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.125:0/1675059636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:23.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:23.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:23.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 106862 106846 76.7323 45.0625 0.024947 0.0130269 2023-10-09T06:35:23.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:23.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:23.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:23.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:23.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:24.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:24.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:24.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:24.244 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:35:24.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:24.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:24.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:24.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:24 smithi125 bash[77299]: cluster 2023-10-09T06:35:23.146633+0000 mgr.y (mgr.25723) 920 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T06:35:24.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:24 smithi125 bash[77299]: audit 2023-10-09T06:35:23.483475+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:24.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:24 smithi125 bash[80002]: cluster 2023-10-09T06:35:23.146633+0000 mgr.y (mgr.25723) 920 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T06:35:24.465 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:24 smithi125 bash[80002]: audit 2023-10-09T06:35:23.483475+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:24.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:24.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:24 smithi155 bash[59495]: cluster 2023-10-09T06:35:23.146633+0000 mgr.y (mgr.25723) 920 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-09T06:35:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:24 smithi155 bash[59495]: audit 2023-10-09T06:35:23.483475+0000 mon.a (mon.0) 1116 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:24.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:24.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:24.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:24.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 107621 107605 76.3994 47.4375 0.0262995 0.0130834 2023-10-09T06:35:24.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:24.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:24.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:24.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:35:25.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:25.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:25.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:25.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:25.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: cluster 2023-10-09T06:35:24.226724+0000 mon.a (mon.0) 1117 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: audit 2023-10-09T06:35:24.232386+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: cluster 2023-10-09T06:35:24.232447+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: cluster 2023-10-09T06:35:24.239263+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: cluster 2023-10-09T06:35:24.243108+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.334 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:25 smithi155 bash[59495]: cluster 2023-10-09T06:35:24.251043+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.354 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:25.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:25.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:25.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: cluster 2023-10-09T06:35:24.226724+0000 mon.a (mon.0) 1117 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: audit 2023-10-09T06:35:24.232386+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: cluster 2023-10-09T06:35:24.232447+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: cluster 2023-10-09T06:35:24.239263+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: cluster 2023-10-09T06:35:24.243108+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[77299]: cluster 2023-10-09T06:35:24.251043+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: cluster 2023-10-09T06:35:24.226724+0000 mon.a (mon.0) 1117 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: audit 2023-10-09T06:35:24.232386+0000 mon.a (mon.0) 1118 : audit 1 from='client.? 172.21.15.125:0/1322305381' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: cluster 2023-10-09T06:35:24.232447+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: cluster 2023-10-09T06:35:24.239263+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: cluster 2023-10-09T06:35:24.243108+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:25 smithi125 bash[80002]: cluster 2023-10-09T06:35:24.251043+0000 osd.4 (osd.4) 126 : cluster 3 failed to encode map e200 with expected crc 2023-10-09T06:35:25.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:25.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:25.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:25.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:25.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:25.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 108442 108426 76.1175 51.3125 0.00503248 0.0131316 2023-10-09T06:35:25.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:25.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:25.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:25.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:26.000 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:26.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:26.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:26.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:26.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:26.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:26 smithi125 bash[77299]: cluster 2023-10-09T06:35:25.147422+0000 mgr.y (mgr.25723) 921 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T06:35:26.372 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:26 smithi125 bash[80002]: cluster 2023-10-09T06:35:25.147422+0000 mgr.y (mgr.25723) 921 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T06:35:26.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:26 smithi155 bash[59495]: cluster 2023-10-09T06:35:25.147422+0000 mgr.y (mgr.25723) 921 : cluster 0 pgmap v932: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s 2023-10-09T06:35:26.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:26.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:26.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:26.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0111 2023-10-09T06:35:26.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 109065 2023-10-09T06:35:26.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:35:26.809 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 75.7302 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 30.7456 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 176.062 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 41.75 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1211 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 491.929 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2817 2023-10-09T06:35:26.810 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 668 2023-10-09T06:35:26.811 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0132002 2023-10-09T06:35:26.811 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0138142 2023-10-09T06:35:26.811 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 0.352992 2023-10-09T06:35:26.811 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00131043 2023-10-09T06:35:26.817 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:35:26.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:26.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:26.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:27.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:27.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:27.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:27.351 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.385 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:27.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:27.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:27.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:27.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:27.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109833 8.4 8:23105463:::benchmark_data_smithi125_176369_object765:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.719739+0000", 2023-10-09T06:35:27.745 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.022229823999999999, 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022273042999999999, 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.746 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109833 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.747 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719739+0000", 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719739+0000", 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719737+0000", 2023-10-09T06:35:27.748 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0480000000000004e-06 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719747+0000", 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3597999999999999e-05 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719760+0000", 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3799999999999999e-06 2023-10-09T06:35:27.749 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719767+0000", 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8140000000000002e-05 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719825+0000", 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103279 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719928+0000", 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4036000000000006e-05 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.720022+0000", 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014546127000000001 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.734568+0000", 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8481999999999999e-05 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.734597+0000", 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014857235 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109834 8.8 8:1f2fcc2e:::benchmark_data_smithi125_176369_object766:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.719792+0000", 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.022177555000000002, 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022321992999999998, 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.753 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109834 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719792+0000", 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719792+0000", 2023-10-09T06:35:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:27.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719791+0000", 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.579e-06 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719795+0000", 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9789999999999996e-06 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719802+0000", 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1480000000000003e-06 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719808+0000", 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00083515599999999998 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.720643+0000", 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116754 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.720760+0000", 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012298699999999999 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.758 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.720883+0000", 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011871061 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.732754+0000", 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012962009 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.759 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109835 8.8 8:15971342:::benchmark_data_smithi125_176369_object767:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.719817+0000", 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.022152614000000001, 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.022381906, 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.760 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109835 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719817+0000", 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719817+0000", 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719816+0000", 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7810000000000001e-06 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.762 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719819+0000", 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9100000000000002e-06 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719825+0000", 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.754e-06 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.719829+0000", 2023-10-09T06:35:27.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0085452749999999997 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.728374+0000", 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142519 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.728516+0000", 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100846 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.728617+0000", 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.008800591 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.765 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109838 8.4 8:29fa5799:::benchmark_data_smithi125_176369_object770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.721092+0000", 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.020876834, 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021185111999999999, 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.766 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109838 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721092+0000", 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721092+0000", 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721091+0000", 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8640000000000001e-06 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721097+0000", 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7860000000000008e-06 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721107+0000", 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9530000000000002e-06 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.721114+0000", 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.009715708 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.730830+0000", 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000115847 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.730945+0000", 2023-10-09T06:35:27.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012493700000000001 2023-10-09T06:35:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:35:27.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.731070+0000", 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052008009999999997 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736271+0000", 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015178806 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.773 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45383.0:109839 8.2 e200/192 8:4ef75d2f:::benchmark_data_smithi125_176369_object771:head v 200'6950, mlcod=200'6946)", 2023-10-09T06:35:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.723444+0000", 2023-10-09T06:35:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.018525501, 2023-10-09T06:35:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018948350999999999, 2023-10-09T06:35:27.774 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723444+0000", 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.775 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723444+0000", 2023-10-09T06:35:27.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:35:27.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723443+0000", 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723452+0000", 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7751e-05 2023-10-09T06:35:27.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723470+0000", 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1359999999999999e-06 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.723477+0000", 2023-10-09T06:35:27.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00091978100000000005 2023-10-09T06:35:27.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.724397+0000", 2023-10-09T06:35:27.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8884000000000001e-05 2023-10-09T06:35:27.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.724416+0000", 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00097204500000000005 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.781 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.782 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45383.0:109843 8.3 e200/192 8:c838afb1:::benchmark_data_smithi125_176369_object775:head v 200'6776, mlcod=200'6774)", 2023-10-09T06:35:27.783 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.725230+0000", 2023-10-09T06:35:27.783 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.016739605000000001, 2023-10-09T06:35:27.783 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017238847000000002, 2023-10-09T06:35:27.783 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.783 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725230+0000", 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725230+0000", 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.784 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725229+0000", 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0414000000000001e-05 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725239+0000", 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2129999999999997e-06 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.785 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725248+0000", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8500000000000002e-06 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725253+0000", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5278999999999998e-05 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725298+0000", 2023-10-09T06:35:27.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5216e-05 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.725314+0000", 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3950999999999996e-05 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109850 8.8 8:1d683a93:::benchmark_data_smithi125_176369_object782:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.787 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.736236+0000", 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0057330100000000002, 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063029319999999998, 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109850 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736236+0000", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736236+0000", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736235+0000", 2023-10-09T06:35:27.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2669999999999997e-06 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736240+0000", 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0199e-05 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736250+0000", 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4790000000000004e-06 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736256+0000", 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036374659999999998 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.739893+0000", 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8042999999999997e-05 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.739981+0000", 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3739000000000006e-05 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740065+0000", 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038286710000000001 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109851 8.9 8:98d2834e:::benchmark_data_smithi125_176369_object783:head [delete] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.736272+0000", 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0056974770000000003, 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063549449999999999, 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109851 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.793 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736272+0000", 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736272+0000", 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736271+0000", 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7939999999999998e-06 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736274+0000", 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.348e-06 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736280+0000", 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1439999999999996e-06 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736284+0000", 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6860999999999996e-05 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736351+0000", 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117748 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736469+0000", 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012655500000000001 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736596+0000", 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045775729999999997 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.741173+0000", 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049015380000000004 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45383.0:109849 8.a e200/192 8:5a5eabaa:::benchmark_data_smithi125_176369_object781:head v 200'6912, mlcod=200'6910)", 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.736412+0000", 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0055570530000000002, 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0062969849999999997, 2023-10-09T06:35:27.798 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736412+0000", 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736412+0000", 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:35:27.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736411+0000", 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0738e-05 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736422+0000", 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9219999999999992e-06 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736432+0000", 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6500000000000001e-06 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736438+0000", 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2638000000000006e-05 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736500+0000", 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.9318999999999999e-05 2023-10-09T06:35:27.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.736520+0000", 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010744800000000001 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:109852 8.8 8.73436d48 (undecoded) ondisk+write+known_if_redirected e200)", 2023-10-09T06:35:27.802 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.738803+0000", 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.003166121, 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039665179999999996, 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "queued for pg", 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 109852 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738803+0000", 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738803+0000", 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738801+0000", 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3959999999999995e-06 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738811+0000", 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3424000000000001e-05 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738824+0000", 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9070000000000007e-06 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.738832+0000", 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.8918999999999999e-05 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45383.0:109853 8.a e200/192 8:58c3242e:::benchmark_data_smithi125_176369_object785:head v 200'6913, mlcod=200'6910)", 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:27.740759+0000", 2023-10-09T06:35:27.806 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001209796, 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020722520000000001, 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740759+0000", 2023-10-09T06:35:27.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740759+0000", 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740758+0000", 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0214000000000001e-05 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740769+0000", 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.137e-05 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740780+0000", 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8340000000000001e-06 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:35:27.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740787+0000", 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5303999999999998e-05 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740842+0000", 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8754e-05 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:27.740861+0000", 2023-10-09T06:35:27.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000101421 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 11 2023-10-09T06:35:27.811 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:35:27.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:27.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:27.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:27.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:27.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:28.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:28.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:28.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:28.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:28.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:28.360 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:28 smithi155 bash[59495]: cluster 2023-10-09T06:35:27.148026+0000 mgr.y (mgr.25723) 922 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-10-09T06:35:28.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:28.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:28.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:28.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.515 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:28 smithi125 bash[77299]: cluster 2023-10-09T06:35:27.148026+0000 mgr.y (mgr.25723) 922 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-10-09T06:35:28.515 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:28 smithi125 bash[80002]: cluster 2023-10-09T06:35:27.148026+0000 mgr.y (mgr.25723) 922 : cluster 0 pgmap v933: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s 2023-10-09T06:35:28.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:28.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:28.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:28.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:28.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:35:28.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:35:28.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:28.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:35:28.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:28.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:28.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:28.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:29.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:29.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:29.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:29.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:29.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:29.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:29.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:29.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:29.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:29.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:29.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:29.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:29.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:29.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:29.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:29.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:29.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:29.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:29.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:30.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:30.024 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:29 smithi125 bash[80002]: cluster 2023-10-09T06:35:29.148632+0000 mgr.y (mgr.25723) 923 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 773 op/s 2023-10-09T06:35:30.024 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:29 smithi125 bash[77299]: cluster 2023-10-09T06:35:29.148632+0000 mgr.y (mgr.25723) 923 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 773 op/s 2023-10-09T06:35:30.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:30.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:30.061 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:35:30.061 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:35:30.061 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:35:30.061 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-09T06:35:30.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-09T06:35:30.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:30.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:30.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:30.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:30.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:29 smithi155 bash[59495]: cluster 2023-10-09T06:35:29.148632+0000 mgr.y (mgr.25723) 923 : cluster 0 pgmap v934: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 773 op/s 2023-10-09T06:35:30.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:30.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:30.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:30.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:30.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:30.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:30.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:30.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:30.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:30.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:30.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:30.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:31.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:31.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:31.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:31.254 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:31.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:31.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:31.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:31] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:35:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:31.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:31.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:31.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:31.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:31.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:31.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:31.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:31.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:31.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:31.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:31.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:32.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:32.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:32.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:32.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:32.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:32.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:32.320 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:32 smithi155 bash[59495]: cluster 2023-10-09T06:35:31.149310+0000 mgr.y (mgr.25723) 924 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 775 op/s 2023-10-09T06:35:32.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:32.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:32.456 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:32 smithi125 bash[77299]: cluster 2023-10-09T06:35:31.149310+0000 mgr.y (mgr.25723) 924 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 775 op/s 2023-10-09T06:35:32.457 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:32 smithi125 bash[80002]: cluster 2023-10-09T06:35:31.149310+0000 mgr.y (mgr.25723) 924 : cluster 0 pgmap v935: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 775 op/s 2023-10-09T06:35:32.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:32.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:32.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:32.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:32.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:32.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:32.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:32.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:32.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:33.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:33.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:33.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:33.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:33.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:33.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:33.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:33.543 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:33.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:33.634 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:33.634 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:33.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:33.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:33.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:33.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:33.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:34.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:34.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:34.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[80002]: cluster 2023-10-09T06:35:33.150034+0000 mgr.y (mgr.25723) 925 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 776 op/s 2023-10-09T06:35:34.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[80002]: audit 2023-10-09T06:35:33.595871+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:34.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[80002]: audit 2023-10-09T06:35:33.598873+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:34.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:34.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:34.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:34.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[77299]: cluster 2023-10-09T06:35:33.150034+0000 mgr.y (mgr.25723) 925 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 776 op/s 2023-10-09T06:35:34.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[77299]: audit 2023-10-09T06:35:33.595871+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:34.488 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:34 smithi125 bash[77299]: audit 2023-10-09T06:35:33.598873+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:34.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:34.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:34.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:34 smithi155 bash[59495]: cluster 2023-10-09T06:35:33.150034+0000 mgr.y (mgr.25723) 925 : cluster 0 pgmap v936: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 776 op/s 2023-10-09T06:35:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:34 smithi155 bash[59495]: audit 2023-10-09T06:35:33.595871+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:34 smithi155 bash[59495]: audit 2023-10-09T06:35:33.598873+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:34.646 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:34.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:34.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:34.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:34.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:34.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:35.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:35.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:35.179 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:35.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:35.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:35.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:35.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:35.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:35.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:35.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:35.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:35.823 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:35.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:35.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:35.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:35.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:36.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:36.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:36.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:36.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:36.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:36 smithi125 bash[77299]: cluster 2023-10-09T06:35:35.152702+0000 mgr.y (mgr.25723) 926 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 93 B/s rd, 22 MiB/s wr, 853 op/s 2023-10-09T06:35:36.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:36 smithi125 bash[80002]: cluster 2023-10-09T06:35:35.152702+0000 mgr.y (mgr.25723) 926 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 93 B/s rd, 22 MiB/s wr, 853 op/s 2023-10-09T06:35:36.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:36.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:36.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:36.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:36 smithi155 bash[59495]: cluster 2023-10-09T06:35:35.152702+0000 mgr.y (mgr.25723) 926 : cluster 0 pgmap v937: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 93 B/s rd, 22 MiB/s wr, 853 op/s 2023-10-09T06:35:36.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:36.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:36.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:36.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:37.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:37.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:37.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:37.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:37.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:37.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:37.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:37 smithi155 bash[59495]: audit 2023-10-09T06:35:36.285141+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:37.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:37.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:37.599 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:37.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:37 smithi125 bash[77299]: audit 2023-10-09T06:35:36.285141+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:37.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:37 smithi125 bash[80002]: audit 2023-10-09T06:35:36.285141+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:37.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:37.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:37.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:37.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:37.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:37.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:37.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:38.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:38.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:38.271 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:38 smithi155 bash[59495]: cluster 2023-10-09T06:35:37.153143+0000 mgr.y (mgr.25723) 927 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 720 op/s 2023-10-09T06:35:38.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:38.361 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:35:38.362 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:35:38.362 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:35:38.362 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:35:38.362 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:35:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:38 smithi125 bash[77299]: cluster 2023-10-09T06:35:37.153143+0000 mgr.y (mgr.25723) 927 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 720 op/s 2023-10-09T06:35:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:38 smithi125 bash[80002]: cluster 2023-10-09T06:35:37.153143+0000 mgr.y (mgr.25723) 927 : cluster 0 pgmap v938: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 720 op/s 2023-10-09T06:35:38.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:38.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:38.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:38.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:38.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:38.828 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:38.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:38.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:38.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:39.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:39.093 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:39.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:39.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:39.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:39.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:39.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:39.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:39.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:39.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:35:39.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:39.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:39.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:39.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:39.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:39.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:39.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:39.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:39.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:40.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:40.026 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:39 smithi125 bash[77299]: cluster 2023-10-09T06:35:39.153723+0000 mgr.y (mgr.25723) 928 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.0 MiB/s wr, 810 op/s 2023-10-09T06:35:40.026 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:39 smithi125 bash[80002]: cluster 2023-10-09T06:35:39.153723+0000 mgr.y (mgr.25723) 928 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.0 MiB/s wr, 810 op/s 2023-10-09T06:35:40.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:40.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:40.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:39 smithi155 bash[59495]: cluster 2023-10-09T06:35:39.153723+0000 mgr.y (mgr.25723) 928 : cluster 0 pgmap v939: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.0 MiB/s wr, 810 op/s 2023-10-09T06:35:40.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:40.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:40.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.444 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:40.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:40.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:40.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:40.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:40.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:40.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:40.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:40.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:40.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:41.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:41.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:41.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:41.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:41.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:41.373 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:41] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:35:41.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:41.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:41.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:41.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:41.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:41.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:41.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:41.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:41.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:41.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:41.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:42.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:42.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:42.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:42.216 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:42 smithi155 bash[59495]: cluster 2023-10-09T06:35:41.154414+0000 mgr.y (mgr.25723) 929 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 795 op/s 2023-10-09T06:35:42.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:42.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:42.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:42.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:42.457 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:42 smithi125 bash[77299]: cluster 2023-10-09T06:35:41.154414+0000 mgr.y (mgr.25723) 929 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 795 op/s 2023-10-09T06:35:42.458 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:42 smithi125 bash[80002]: cluster 2023-10-09T06:35:41.154414+0000 mgr.y (mgr.25723) 929 : cluster 0 pgmap v940: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.8 MiB/s wr, 795 op/s 2023-10-09T06:35:42.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:42.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:42.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:42.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:42.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:42.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:43.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:43.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:43.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:43.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:43.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:43.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:43.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:43.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:43.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:43.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:43.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:43.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:43.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:43.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:43.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:43.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:43.917 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:35:43.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:43.992 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:35:43.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:44.040 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:35:44.071 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:44.101 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:35:44.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:44.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:44.143 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:35:44.183 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:35:44.221 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:35:44.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:44.260 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:35:44.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.266 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:35:44.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: cluster 2023-10-09T06:35:43.157001+0000 mgr.y (mgr.25723) 930 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:35:44.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: audit 2023-10-09T06:35:43.613392+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: audit 2023-10-09T06:35:43.631144+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: audit 2023-10-09T06:35:43.843165+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.125:0/717426592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:44.299 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: audit 2023-10-09T06:35:43.876389+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.125:0/3115656494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[77299]: audit 2023-10-09T06:35:43.877567+0000 mon.a (mon.0) 1126 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: cluster 2023-10-09T06:35:43.157001+0000 mgr.y (mgr.25723) 930 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: audit 2023-10-09T06:35:43.613392+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: audit 2023-10-09T06:35:43.631144+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: audit 2023-10-09T06:35:43.843165+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.125:0/717426592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: audit 2023-10-09T06:35:43.876389+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.125:0/3115656494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.300 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:44 smithi125 bash[80002]: audit 2023-10-09T06:35:43.877567+0000 mon.a (mon.0) 1126 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: cluster 2023-10-09T06:35:43.157001+0000 mgr.y (mgr.25723) 930 : cluster 0 pgmap v941: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: audit 2023-10-09T06:35:43.613392+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: audit 2023-10-09T06:35:43.631144+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: audit 2023-10-09T06:35:43.843165+0000 mon.a (mon.0) 1125 : audit 0 from='client.? 172.21.15.125:0/717426592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: audit 2023-10-09T06:35:43.876389+0000 mon.c (mon.2) 90 : audit 1 from='client.? 172.21.15.125:0/3115656494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:44 smithi155 bash[59495]: audit 2023-10-09T06:35:43.877567+0000 mon.a (mon.0) 1126 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:44.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:44.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:44.531 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:44.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:44.655 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:35:44.655 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-09T06:35:44.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:44.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:44.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:44.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:44.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:44.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:44.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:45.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:45.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:45.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:45.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:45.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:45.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: audit 2023-10-09T06:35:44.228768+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: audit 2023-10-09T06:35:44.233189+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.125:0/2076513548' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.627479+0000 mon.a (mon.0) 1128 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: audit 2023-10-09T06:35:44.633970+0000 mon.a (mon.0) 1129 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: audit 2023-10-09T06:35:44.634326+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.634584+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.651133+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.652535+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: audit 2023-10-09T06:35:44.228768+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: audit 2023-10-09T06:35:44.233189+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.125:0/2076513548' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.627479+0000 mon.a (mon.0) 1128 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: audit 2023-10-09T06:35:44.633970+0000 mon.a (mon.0) 1129 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: audit 2023-10-09T06:35:44.634326+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.634584+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.651133+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:45 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.652535+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:45.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:45.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: audit 2023-10-09T06:35:44.228768+0000 mon.a (mon.0) 1127 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: audit 2023-10-09T06:35:44.233189+0000 mon.b (mon.1) 24 : audit 1 from='client.? 172.21.15.125:0/2076513548' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.627479+0000 mon.a (mon.0) 1128 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: audit 2023-10-09T06:35:44.633970+0000 mon.a (mon.0) 1129 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: audit 2023-10-09T06:35:44.634326+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.634584+0000 mon.a (mon.0) 1131 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.651133+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:45 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.652535+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:45.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:45.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:45.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:45.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:45.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:45.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:46.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:46.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.647476+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.648997+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:44.651240+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.157563+0000 mgr.y (mgr.25723) 931 : cluster 0 pgmap v943: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.630732+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.644209+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.650597+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.224 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.654369+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.656957+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.658852+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.647476+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.648997+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:44.651240+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.157563+0000 mgr.y (mgr.25723) 931 : cluster 0 pgmap v943: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.630732+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.644209+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:35:46.225 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.650597+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.654369+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.656957+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.226 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:46 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.658852+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:46.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:46.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:46.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.647476+0000 osd.4 (osd.4) 127 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.648997+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:44.651240+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e201 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.157563+0000 mgr.y (mgr.25723) 931 : cluster 0 pgmap v943: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.630732+0000 mon.a (mon.0) 1132 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.644209+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.650597+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.654369+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.656957+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.359 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:46 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.658852+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:46.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:46.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:46.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:46.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:46.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:46.681 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:46.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:46.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:46.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:46.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:46.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:46.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:47.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:47.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:47.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:47.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:47.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:47.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:47.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:47 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.649828+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:47 smithi125 bash[77299]: cluster 2023-10-09T06:35:45.654046+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:47 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.649828+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:47 smithi125 bash[80002]: cluster 2023-10-09T06:35:45.654046+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:47.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:47.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:47.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:47.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:47 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.649828+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:47 smithi155 bash[59495]: cluster 2023-10-09T06:35:45.654046+0000 osd.4 (osd.4) 128 : cluster 3 failed to encode map e202 with expected crc 2023-10-09T06:35:47.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:47.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:47.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:47.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:47.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:47.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:48.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:48.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:48.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:48.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:48.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:48.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:48.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:48.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:48 smithi125 bash[77299]: cluster 2023-10-09T06:35:47.162436+0000 mgr.y (mgr.25723) 932 : cluster 0 pgmap v945: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:35:48.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:48 smithi125 bash[80002]: cluster 2023-10-09T06:35:47.162436+0000 mgr.y (mgr.25723) 932 : cluster 0 pgmap v945: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:35:48.421 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:48.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:48.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:48 smithi155 bash[59495]: cluster 2023-10-09T06:35:47.162436+0000 mgr.y (mgr.25723) 932 : cluster 0 pgmap v945: 89 pgs: 5 peering, 84 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 767 op/s 2023-10-09T06:35:48.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:48.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:48.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:48.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:48.794 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:48.795 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:48.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:48.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:48.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:48.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:48.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:49.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:35:49.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:49.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:49.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:49.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:49.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:49.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:49.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:49.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:35:49.819 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:49 smithi155 bash[59495]: audit 2023-10-09T06:35:48.596307+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:49.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:49 smithi155 bash[59495]: audit 2023-10-09T06:35:48.598032+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:49.820 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:49 smithi155 bash[59495]: cluster 2023-10-09T06:35:49.163616+0000 mgr.y (mgr.25723) 933 : cluster 0 pgmap v946: 89 pgs: 5 peering, 84 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:35:49.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.829 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[80002]: audit 2023-10-09T06:35:48.596307+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:49.829 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[80002]: audit 2023-10-09T06:35:48.598032+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:49.829 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[80002]: cluster 2023-10-09T06:35:49.163616+0000 mgr.y (mgr.25723) 933 : cluster 0 pgmap v946: 89 pgs: 5 peering, 84 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:35:49.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[77299]: audit 2023-10-09T06:35:48.596307+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:49.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[77299]: audit 2023-10-09T06:35:48.598032+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:49.830 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:49 smithi125 bash[77299]: cluster 2023-10-09T06:35:49.163616+0000 mgr.y (mgr.25723) 933 : cluster 0 pgmap v946: 89 pgs: 5 peering, 84 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:35:49.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:49.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:35:49.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:49.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:49.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:50.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:50.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:50.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:50.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:50.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:50.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:50.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:50.549 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:50.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:50.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:50.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:50.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:50.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:50.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:50 smithi155 bash[59495]: audit 2023-10-09T06:35:49.863483+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:50.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:50 smithi155 bash[59495]: audit 2023-10-09T06:35:49.873624+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:50.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:50 smithi155 bash[59495]: audit 2023-10-09T06:35:50.501129+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:35:50.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:50 smithi155 bash[59495]: audit 2023-10-09T06:35:50.502472+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:35:50.936 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:50 smithi155 bash[59495]: audit 2023-10-09T06:35:50.510858+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:50.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:50.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:51.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:51.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[77299]: audit 2023-10-09T06:35:49.863483+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[77299]: audit 2023-10-09T06:35:49.873624+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[77299]: audit 2023-10-09T06:35:50.501129+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[77299]: audit 2023-10-09T06:35:50.502472+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[77299]: audit 2023-10-09T06:35:50.510858+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[80002]: audit 2023-10-09T06:35:49.863483+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[80002]: audit 2023-10-09T06:35:49.873624+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[80002]: audit 2023-10-09T06:35:50.501129+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:35:51.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[80002]: audit 2023-10-09T06:35:50.502472+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:35:51.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:50 smithi125 bash[80002]: audit 2023-10-09T06:35:50.510858+0000 mon.a (mon.0) 1140 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:35:51.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:51.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:51.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:51.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:51.475 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:35:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:35:51] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-10-09T06:35:51.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:51.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:51.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:51.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:51.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:51 smithi125 bash[77299]: cluster 2023-10-09T06:35:51.165696+0000 mgr.y (mgr.25723) 934 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:51.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:51.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:52.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:52.026 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:51 smithi155 bash[59495]: cluster 2023-10-09T06:35:51.165696+0000 mgr.y (mgr.25723) 934 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:52.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:52.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:51 smithi125 bash[80002]: cluster 2023-10-09T06:35:51.165696+0000 mgr.y (mgr.25723) 934 : cluster 0 pgmap v947: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:52.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:52.291 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:52.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:52.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:52.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:52.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:52.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:52.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:52.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:52.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:52.886 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:52 smithi155 bash[59495]: cluster 2023-10-09T06:35:51.863001+0000 mon.a (mon.0) 1141 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T06:35:52.927 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:35:52.927 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:35:52.927 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T06:35:52.927 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:35:52.927 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:35:52.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T06:35:52.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:52.985 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:52 smithi125 bash[77299]: cluster 2023-10-09T06:35:51.863001+0000 mon.a (mon.0) 1141 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T06:35:52.985 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:52 smithi125 bash[80002]: cluster 2023-10-09T06:35:51.863001+0000 mon.a (mon.0) 1141 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T06:35:53.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:53.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:53.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:53.264 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:53.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:53.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:53.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:53.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:53.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:53.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:53.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:53.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:53.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:53.892 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:53.893 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:53.931 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:53 smithi125 bash[77299]: cluster 2023-10-09T06:35:53.167002+0000 mgr.y (mgr.25723) 935 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:53.931 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:53 smithi125 bash[80002]: cluster 2023-10-09T06:35:53.167002+0000 mgr.y (mgr.25723) 935 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:53.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:53.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:53.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:54.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:54.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:54.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:54.216 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:53 smithi155 bash[59495]: cluster 2023-10-09T06:35:53.167002+0000 mgr.y (mgr.25723) 935 : cluster 0 pgmap v948: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T06:35:54.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:54.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:54.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:54.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:54.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:54.626 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:35:54.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:54.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:54.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:54.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:54.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:54.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:54.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:54.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:54.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:55.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.054 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:55.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:55.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:55.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:55.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:55.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:55.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:55.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:55.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:55.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:55.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:55.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:55.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:55.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:56.101 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:56.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:56.267 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:56.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:56.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:56.518 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:56 smithi155 bash[59495]: cluster 2023-10-09T06:35:55.167722+0000 mgr.y (mgr.25723) 936 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:35:56.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:56.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:56.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:56 smithi125 bash[77299]: cluster 2023-10-09T06:35:55.167722+0000 mgr.y (mgr.25723) 936 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:35:56.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:56 smithi125 bash[80002]: cluster 2023-10-09T06:35:55.167722+0000 mgr.y (mgr.25723) 936 : cluster 0 pgmap v949: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:35:56.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:56.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:56.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:56.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:56.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:56.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.941 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:56.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:56.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:57.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:57.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:57.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:57.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:57.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:57.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:57.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:57.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:57.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:57.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:57.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:57.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:57.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:58.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:58.237 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:35:58 smithi125 bash[77299]: cluster 2023-10-09T06:35:57.168256+0000 mgr.y (mgr.25723) 937 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:35:58.237 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:35:58 smithi125 bash[80002]: cluster 2023-10-09T06:35:57.168256+0000 mgr.y (mgr.25723) 937 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:35:58.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:58.274 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:35:58 smithi155 bash[59495]: cluster 2023-10-09T06:35:57.168256+0000 mgr.y (mgr.25723) 937 : cluster 0 pgmap v950: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:35:58.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:58.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:58.526 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:58.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:58.847 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:58.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:58.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:58.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:59.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:59.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:59.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:59.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:59.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:59.287 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:59.392 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:59.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:59.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:59.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:59.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:59.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:59.753 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:59.753 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:59.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:59.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:59.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:59.939 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:35:59.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:00.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:00.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:00.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:00.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:00.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:00.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:00.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:00 smithi155 bash[59495]: audit 2023-10-09T06:35:59.089824+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:00.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:00 smithi155 bash[59495]: cluster 2023-10-09T06:35:59.168726+0000 mgr.y (mgr.25723) 938 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:36:00.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:00 smithi125 bash[77299]: audit 2023-10-09T06:35:59.089824+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:00.407 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:00 smithi125 bash[77299]: cluster 2023-10-09T06:35:59.168726+0000 mgr.y (mgr.25723) 938 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:36:00.407 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:00 smithi125 bash[80002]: audit 2023-10-09T06:35:59.089824+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:00.407 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:00 smithi125 bash[80002]: cluster 2023-10-09T06:35:59.168726+0000 mgr.y (mgr.25723) 938 : cluster 0 pgmap v951: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:36:00.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:00.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:00.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:00.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:00.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:00.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:36:00.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:00.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:00.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:00.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:01.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:01.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:01.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:01.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:01.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:01.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:01.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:01.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:01.554 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:01] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.33.4" 2023-10-09T06:36:01.584 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:01.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:01.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:01.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:01.849 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:01.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:01.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:01.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:01.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:02.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:02.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:02.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:02.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:02.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:02.372 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:02 smithi155 bash[59495]: cluster 2023-10-09T06:36:01.169185+0000 mgr.y (mgr.25723) 939 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T06:36:02.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:02.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:02.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:02 smithi125 bash[77299]: cluster 2023-10-09T06:36:01.169185+0000 mgr.y (mgr.25723) 939 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T06:36:02.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:02 smithi125 bash[80002]: cluster 2023-10-09T06:36:01.169185+0000 mgr.y (mgr.25723) 939 : cluster 0 pgmap v952: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-09T06:36:02.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:02.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:02.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:02.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:02.809 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:02.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:02.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:02.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:03.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:03.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:03.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:03.199 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:03.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:03.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:03.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:03.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:03.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:03.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:03.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:03.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:03.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:03.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:03.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:03.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:03.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:03.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:03.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:03.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:04.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:04.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:04.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:04.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:04.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[77299]: cluster 2023-10-09T06:36:03.169692+0000 mgr.y (mgr.25723) 940 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:36:04.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[77299]: audit 2023-10-09T06:36:03.594744+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:04.242 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[77299]: audit 2023-10-09T06:36:03.599688+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:04.242 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[80002]: cluster 2023-10-09T06:36:03.169692+0000 mgr.y (mgr.25723) 940 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:36:04.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[80002]: audit 2023-10-09T06:36:03.594744+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:04.243 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:04 smithi125 bash[80002]: audit 2023-10-09T06:36:03.599688+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:04.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:04.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:04.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:04 smithi155 bash[59495]: cluster 2023-10-09T06:36:03.169692+0000 mgr.y (mgr.25723) 940 : cluster 0 pgmap v953: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:36:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:04 smithi155 bash[59495]: audit 2023-10-09T06:36:03.594744+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:04 smithi155 bash[59495]: audit 2023-10-09T06:36:03.599688+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:04.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:04.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:04.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:04.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:04.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:04.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:04.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:04.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:04.834 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:04.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:04.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:04.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:05.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:05.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:05.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:05.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:05.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:05.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:05.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:05.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:05.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:05.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:05.715 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:05.782 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:05.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:05.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:05.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:05.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:06.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:06.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:06.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:06.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:06.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:06.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:06.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:06.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:06.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:06.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:06 smithi155 bash[59495]: cluster 2023-10-09T06:36:05.170197+0000 mgr.y (mgr.25723) 941 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-09T06:36:06.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:06.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:06 smithi125 bash[77299]: cluster 2023-10-09T06:36:05.170197+0000 mgr.y (mgr.25723) 941 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-09T06:36:06.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:06 smithi125 bash[80002]: cluster 2023-10-09T06:36:05.170197+0000 mgr.y (mgr.25723) 941 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-09T06:36:06.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:06.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:06.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:06.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.894 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:06.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:07.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:07.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:07.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:07.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:07.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:07.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.444 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:07.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:07.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:07.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:07.646 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:36:07.676 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:07.715 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:36:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:07.766 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:36:07.810 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:36:07.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:07.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:07.853 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:36:07.893 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:36:07.937 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:36:07.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:07.969 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.976 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.977 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:07.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:07.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:08.005 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:36:08.005 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:36:08.005 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:36:08.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:08.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:08.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:08.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:08.270 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:36:08.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[77299]: cluster 2023-10-09T06:36:07.170747+0000 mgr.y (mgr.25723) 942 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:36:08.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[77299]: audit 2023-10-09T06:36:07.568230+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.125:0/814773374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:08.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[77299]: audit 2023-10-09T06:36:07.914967+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.125:0/3842719176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[77299]: audit 2023-10-09T06:36:07.915650+0000 mon.a (mon.0) 1145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.315 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[80002]: cluster 2023-10-09T06:36:07.170747+0000 mgr.y (mgr.25723) 942 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:36:08.315 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[80002]: audit 2023-10-09T06:36:07.568230+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.125:0/814773374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:08.315 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[80002]: audit 2023-10-09T06:36:07.914967+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.125:0/3842719176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.315 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:08 smithi125 bash[80002]: audit 2023-10-09T06:36:07.915650+0000 mon.a (mon.0) 1145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:08 smithi155 bash[59495]: cluster 2023-10-09T06:36:07.170747+0000 mgr.y (mgr.25723) 942 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-09T06:36:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:08 smithi155 bash[59495]: audit 2023-10-09T06:36:07.568230+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.125:0/814773374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:08 smithi155 bash[59495]: audit 2023-10-09T06:36:07.914967+0000 mon.c (mon.2) 91 : audit 1 from='client.? 172.21.15.125:0/3842719176' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:08 smithi155 bash[59495]: audit 2023-10-09T06:36:07.915650+0000 mon.a (mon.0) 1145 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:08.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:08.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:08.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:08.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:08.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:08.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:08.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:08.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:09.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:09.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:09.197 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:09.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[77299]: cluster 2023-10-09T06:36:08.243789+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[77299]: audit 2023-10-09T06:36:08.253078+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[77299]: cluster 2023-10-09T06:36:08.253312+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:36:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[77299]: cluster 2023-10-09T06:36:08.254186+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[77299]: cluster 2023-10-09T06:36:08.254948+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[80002]: cluster 2023-10-09T06:36:08.243789+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[80002]: audit 2023-10-09T06:36:08.253078+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[80002]: cluster 2023-10-09T06:36:08.253312+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[80002]: cluster 2023-10-09T06:36:08.254186+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:09 smithi125 bash[80002]: cluster 2023-10-09T06:36:08.254948+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.477 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:09.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:09.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:09 smithi155 bash[59495]: cluster 2023-10-09T06:36:08.243789+0000 mon.a (mon.0) 1146 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:09.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:09 smithi155 bash[59495]: audit 2023-10-09T06:36:08.253078+0000 mon.a (mon.0) 1147 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:09.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:09 smithi155 bash[59495]: cluster 2023-10-09T06:36:08.253312+0000 mon.a (mon.0) 1148 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:36:09.588 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:09 smithi155 bash[59495]: cluster 2023-10-09T06:36:08.254186+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.589 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:09 smithi155 bash[59495]: cluster 2023-10-09T06:36:08.254948+0000 osd.4 (osd.4) 129 : cluster 3 failed to encode map e203 with expected crc 2023-10-09T06:36:09.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:09.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:09.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:09.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:09.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:09.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:09.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:10.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:10.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:10.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:10.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:10.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:10.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:10.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:10.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:10 smithi125 bash[77299]: cluster 2023-10-09T06:36:09.171308+0000 mgr.y (mgr.25723) 943 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T06:36:10.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:10 smithi125 bash[80002]: cluster 2023-10-09T06:36:09.171308+0000 mgr.y (mgr.25723) 943 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T06:36:10.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:10.633 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:36:10.633 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:36:10.633 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:36:10.633 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:36:10.633 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:36:10.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:10 smithi155 bash[59495]: cluster 2023-10-09T06:36:09.171308+0000 mgr.y (mgr.25723) 943 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-09T06:36:10.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:10.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:10.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:10.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:10.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:10.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:10.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:11.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:11.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:11.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:11.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:11.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:11.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:11.492 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:11] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:36:11.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:11.623 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:11.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:11.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:11.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:36:11.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:11.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:11.957 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:11.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:12.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:12.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:12.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:12.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:12.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:12.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:12.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:12 smithi125 bash[77299]: cluster 2023-10-09T06:36:11.171923+0000 mgr.y (mgr.25723) 944 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:12.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:12 smithi125 bash[80002]: cluster 2023-10-09T06:36:11.171923+0000 mgr.y (mgr.25723) 944 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:12.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:12.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:12.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:12.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:12.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:12 smithi155 bash[59495]: cluster 2023-10-09T06:36:11.171923+0000 mgr.y (mgr.25723) 944 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:12.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:12.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:12.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:36:12.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:36:12.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:12.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:12.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:13.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:13.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:13.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:13.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:13.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:13.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:13.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:13.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:13.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:13.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:13.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:13.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:14.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:14.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:14.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:14.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:14.284 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:14 smithi125 bash[77299]: cluster 2023-10-09T06:36:13.172462+0000 mgr.y (mgr.25723) 945 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:36:14.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:14.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:14.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:14.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:14.526 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:14.536 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:14 smithi155 bash[59495]: cluster 2023-10-09T06:36:13.172462+0000 mgr.y (mgr.25723) 945 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:36:14.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:14.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:14 smithi125 bash[80002]: cluster 2023-10-09T06:36:13.172462+0000 mgr.y (mgr.25723) 945 : cluster 0 pgmap v959: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-09T06:36:14.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:14.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:14.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:14.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:14.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:14.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:15.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:15.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:15.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:15.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:15.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:15.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:15.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:15.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:15.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:15.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:15.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:15.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:15.832 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:15.832 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:15.832 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:36:15.832 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:36:15.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:15.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:15.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:16.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:16.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:16.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:16.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:16 smithi125 bash[77299]: cluster 2023-10-09T06:36:15.173238+0000 mgr.y (mgr.25723) 946 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:36:16.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:16 smithi125 bash[80002]: cluster 2023-10-09T06:36:15.173238+0000 mgr.y (mgr.25723) 946 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:36:16.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:16.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:16.455 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:16 smithi155 bash[59495]: cluster 2023-10-09T06:36:15.173238+0000 mgr.y (mgr.25723) 946 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:36:16.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:16.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:16.605 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:16.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:16.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:16.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:16.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:16.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:16.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:17.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:17.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:17.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:17.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:17.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:17.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:17.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:17.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:17.679 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:17.684 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:17.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:17.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:17.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:17.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:17.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:17.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:18.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:18.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:18.283 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:18.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:18.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:18.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:18.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:18 smithi155 bash[59495]: cluster 2023-10-09T06:36:17.173681+0000 mgr.y (mgr.25723) 947 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T06:36:18.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:18.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:18 smithi125 bash[77299]: cluster 2023-10-09T06:36:17.173681+0000 mgr.y (mgr.25723) 947 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T06:36:18.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:18 smithi125 bash[80002]: cluster 2023-10-09T06:36:17.173681+0000 mgr.y (mgr.25723) 947 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-09T06:36:18.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:18.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:18.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:18.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:18.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:19.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:19.151 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:19.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:19.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:19.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:19 smithi155 bash[59495]: audit 2023-10-09T06:36:18.591657+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:19.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:19.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.674 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:19 smithi125 bash[77299]: audit 2023-10-09T06:36:18.591657+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:19.675 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:19 smithi125 bash[80002]: audit 2023-10-09T06:36:18.591657+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:19.716 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:19.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:19.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:19.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:20.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:20.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:20.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:20.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:20.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:20 smithi155 bash[59495]: cluster 2023-10-09T06:36:19.174153+0000 mgr.y (mgr.25723) 948 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T06:36:20.579 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:20 smithi125 bash[77299]: cluster 2023-10-09T06:36:19.174153+0000 mgr.y (mgr.25723) 948 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T06:36:20.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:20 smithi125 bash[80002]: cluster 2023-10-09T06:36:19.174153+0000 mgr.y (mgr.25723) 948 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-09T06:36:20.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:20.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:20.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:20.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:20.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:20.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:20.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:20.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:21.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:21.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:21.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:21.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:21.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.458 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:14537 7.7 7:eb153be0:::benchmark_data_smithi125_153312_object14536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:43.734379+0000", 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 457.71253645799999, 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17982935999999999, 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 14537 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.459 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734379+0000", 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734379+0000", 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.460 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734377+0000", 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8250000000000004e-05 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734465+0000", 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.543e-05 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734481+0000", 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3570000000000003e-06 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734487+0000", 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9404999999999999e-05 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734536+0000", 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000169307 2023-10-09T06:36:21.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734706+0000", 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014959700000000001 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734855+0000", 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0018380320000000001 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.736693+0000", 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0324999999999997e-05 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.736744+0000", 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.177348373 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914092+0000", 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7246e-05 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914159+0000", 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8736999999999998e-05 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914208+0000", 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17982935999999999 2023-10-09T06:36:21.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:14544 7.7 7:eb91b996:::benchmark_data_smithi125_153312_object14543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:43.734953+0000", 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 457.71196255000001, 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17942535300000001, 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.466 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 14544 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734953+0000", 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.467 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.468 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734953+0000", 2023-10-09T06:36:21.468 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:36:21.468 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.468 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.468 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.734951+0000", 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2778000000000002e-05 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.469 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735004+0000", 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2901000000000001e-05 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735017+0000", 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9299999999999997e-06 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735024+0000", 2023-10-09T06:36:21.470 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00065535599999999995 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735679+0000", 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112178 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.471 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735792+0000", 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015475299999999999 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.735946+0000", 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0010402479999999999 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.472 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.736987+0000", 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030530599999999998 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737292+0000", 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.176974311 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.473 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914266+0000", 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2041999999999999e-05 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914338+0000", 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9706000000000002e-05 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914378+0000", 2023-10-09T06:36:21.474 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17942535300000001 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:14547 7.7 7:e9fd502f:::benchmark_data_smithi125_153312_object14546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:43.737736+0000", 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 457.70917901299998, 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.176802918, 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.475 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 14547 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737736+0000", 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.476 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737736+0000", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737735+0000", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7759999999999998e-05 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737772+0000", 2023-10-09T06:36:21.477 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5957e-05 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737788+0000", 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0096e-05 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737798+0000", 2023-10-09T06:36:21.478 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5952999999999999e-05 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737844+0000", 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013227700000000001 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.737977+0000", 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000127286 2023-10-09T06:36:21.479 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.738104+0000", 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00070404300000000001 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.738808+0000", 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020941300000000001 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.480 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.739017+0000", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17541800799999999 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914435+0000", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3884000000000001e-05 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914489+0000", 2023-10-09T06:36:21.481 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9719000000000003e-05 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:43.914539+0000", 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.176802918 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.482 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:31425 7.7 7:ecd5e694:::benchmark_data_smithi125_153312_object31424:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e173)", 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 447.36951112899999, 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.210224828, 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 31425 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.483 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.484 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077404+0000", 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077402+0000", 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4017000000000001e-05 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.485 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077456+0000", 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8720999999999999e-05 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077475+0000", 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6710000000000005e-06 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.486 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077483+0000", 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4039000000000003e-05 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077527+0000", 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110596 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077638+0000", 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014253000000000001 2023-10-09T06:36:21.487 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.077780+0000", 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00135721 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.079138+0000", 2023-10-09T06:36:21.488 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036136499999999999 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.082751+0000", 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20473650099999999 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.287488+0000", 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4023999999999995e-05 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.489 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.287562+0000", 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6989999999999994e-05 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:28:54.287629+0000", 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.210224828 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.490 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:40256 7.7 7:ef8ec236:::benchmark_data_smithi125_153312_object40255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:00.868792+0000", 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 440.57812337399997, 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16661725599999999, 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.491 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 40256 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868792+0000", 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.492 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868792+0000", 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868789+0000", 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8436000000000005e-05 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.493 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868878+0000", 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6742e-05 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868894+0000", 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2690000000000005e-06 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.494 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868904+0000", 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2015999999999997e-05 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.868966+0000", 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018372399999999999 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.869149+0000", 2023-10-09T06:36:21.495 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015475999999999999 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.869304+0000", 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001777533 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.871082+0000", 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036146640000000001 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.496 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:00.874696+0000", 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16059974199999999 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.035296+0000", 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5889999999999994e-05 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.497 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.035362+0000", 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7063e-05 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:01.035409+0000", 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16661725599999999 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.498 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:71433 7.7 7:e58d0237:::benchmark_data_smithi125_153312_object71432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:28.015995+0000", 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 413.43092036199999, 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.233150158, 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 71433 2023-10-09T06:36:21.499 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.015995+0000", 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.015995+0000", 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.500 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.015993+0000", 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3132000000000005e-05 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.016056+0000", 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5919999999999999e-05 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.501 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.016082+0000", 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2640000000000002e-06 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.016086+0000", 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041276029999999997 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.020214+0000", 2023-10-09T06:36:21.502 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024301699999999999 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.020457+0000", 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00030943099999999999 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.020766+0000", 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0056277860000000001 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.503 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.026394+0000", 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022395980000000002 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.028633+0000", 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22037245799999999 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.504 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.249006+0000", 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5402000000000007e-05 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.249091+0000", 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3560999999999998e-05 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.249145+0000", 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.233150158 2023-10-09T06:36:21.505 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:71456 7.7 7:e4668216:::benchmark_data_smithi125_153312_object71455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:28.032750+0000", 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 413.41416526500001, 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22487347099999999, 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.506 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 71456 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032750+0000", 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.507 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032750+0000", 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.508 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032748+0000", 2023-10-09T06:36:21.509 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0112999999999999e-05 2023-10-09T06:36:21.509 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.510 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.510 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.511 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032798+0000", 2023-10-09T06:36:21.511 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4352000000000001e-05 2023-10-09T06:36:21.511 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.511 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.511 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032812+0000", 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0430000000000008e-06 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.512 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032820+0000", 2023-10-09T06:36:21.513 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8273000000000003e-05 2023-10-09T06:36:21.513 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.513 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.513 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.513 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.032879+0000", 2023-10-09T06:36:21.514 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014159 2023-10-09T06:36:21.514 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.514 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.514 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.514 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.033020+0000", 2023-10-09T06:36:21.517 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000159557 2023-10-09T06:36:21.517 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.517 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.517 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.033180+0000", 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002739577 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.518 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.035919+0000", 2023-10-09T06:36:21.519 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033788200000000002 2023-10-09T06:36:21.519 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.036257+0000", 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.221221642 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.520 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.521 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.521 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.257479+0000", 2023-10-09T06:36:21.521 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5047000000000001e-05 2023-10-09T06:36:21.521 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.521 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.522 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.522 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.257564+0000", 2023-10-09T06:36:21.522 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9485999999999999e-05 2023-10-09T06:36:21.522 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.522 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.257623+0000", 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22487347099999999 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.523 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.524 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.524 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.524 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:71469 7.7 7:e178850c:::benchmark_data_smithi125_153312_object71468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e175)", 2023-10-09T06:36:21.524 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:29:28.049619+0000", 2023-10-09T06:36:21.524 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 413.39729648899998, 2023-10-09T06:36:21.525 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20879635699999999, 2023-10-09T06:36:21.525 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.525 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.525 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.526 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.526 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 71469 2023-10-09T06:36:21.526 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.526 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.526 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.527 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.527 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049619+0000", 2023-10-09T06:36:21.527 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.527 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.527 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049619+0000", 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.528 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.529 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049616+0000", 2023-10-09T06:36:21.529 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4800000000000003e-05 2023-10-09T06:36:21.529 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.529 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.529 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.530 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049681+0000", 2023-10-09T06:36:21.530 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.421e-05 2023-10-09T06:36:21.530 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.530 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.530 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.531 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049705+0000", 2023-10-09T06:36:21.531 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0520000000000007e-06 2023-10-09T06:36:21.531 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.531 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.531 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049714+0000", 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0275000000000001e-05 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.532 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049775+0000", 2023-10-09T06:36:21.533 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000152709 2023-10-09T06:36:21.533 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.533 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.533 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.533 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.049927+0000", 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017112799999999999 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.050099+0000", 2023-10-09T06:36:21.534 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023513230000000002 2023-10-09T06:36:21.535 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.535 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.535 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.535 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.052450+0000", 2023-10-09T06:36:21.535 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016111899999999999 2023-10-09T06:36:21.536 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.536 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.536 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.536 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.052611+0000", 2023-10-09T06:36:21.536 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20567011499999999 2023-10-09T06:36:21.537 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.537 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.537 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.258281+0000", 2023-10-09T06:36:21.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7367999999999995e-05 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.258358+0000", 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6521999999999999e-05 2023-10-09T06:36:21.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.539 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.539 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.539 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:29:28.258415+0000", 2023-10-09T06:36:21.539 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20879635699999999 2023-10-09T06:36:21.539 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.540 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.540 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.540 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.540 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:113143 7.7 7:e596bcbc:::benchmark_data_smithi125_153312_object8456:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-10-09T06:36:21.541 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:30:18.001741+0000", 2023-10-09T06:36:21.541 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 363.44517395499997, 2023-10-09T06:36:21.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17574356299999999, 2023-10-09T06:36:21.541 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.541 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.542 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.542 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 113143 2023-10-09T06:36:21.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.543 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.543 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.543 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001741+0000", 2023-10-09T06:36:21.543 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001741+0000", 2023-10-09T06:36:21.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.545 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.545 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.545 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001739+0000", 2023-10-09T06:36:21.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.251e-06 2023-10-09T06:36:21.546 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.546 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.546 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.546 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001749+0000", 2023-10-09T06:36:21.547 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7022999999999999e-05 2023-10-09T06:36:21.547 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.547 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.547 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.548 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001766+0000", 2023-10-09T06:36:21.548 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3515e-05 2023-10-09T06:36:21.548 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.548 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.549 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.549 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001779+0000", 2023-10-09T06:36:21.549 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5180999999999998e-05 2023-10-09T06:36:21.549 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.550 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.550 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.550 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.001844+0000", 2023-10-09T06:36:21.550 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015542999999999999 2023-10-09T06:36:21.551 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.551 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.551 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.551 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.002000+0000", 2023-10-09T06:36:21.552 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016762400000000001 2023-10-09T06:36:21.552 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.552 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.552 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.552 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.002167+0000", 2023-10-09T06:36:21.553 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033907730000000001 2023-10-09T06:36:21.553 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.553 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.553 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.554 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.005558+0000", 2023-10-09T06:36:21.554 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020944330000000001 2023-10-09T06:36:21.554 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.554 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.555 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.555 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.007653+0000", 2023-10-09T06:36:21.555 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.16968819900000001 2023-10-09T06:36:21.555 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.556 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.556 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.556 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.177341+0000", 2023-10-09T06:36:21.556 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4967000000000001e-05 2023-10-09T06:36:21.557 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.557 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.557 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.557 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.177436+0000", 2023-10-09T06:36:21.558 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8881000000000002e-05 2023-10-09T06:36:21.558 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.558 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.558 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.559 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:18.177485+0000", 2023-10-09T06:36:21.559 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17574356299999999 2023-10-09T06:36:21.559 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.559 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.560 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.560 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.560 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.560 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:127432 7.7 7:eaca6542:::benchmark_data_smithi125_153312_object22745:head [delete] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-09T06:36:21.561 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:30:37.473783+0000", 2023-10-09T06:36:21.561 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 343.97313228299998, 2023-10-09T06:36:21.561 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.171340835, 2023-10-09T06:36:21.561 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.561 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.562 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.562 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.562 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.562 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 127432 2023-10-09T06:36:21.563 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.563 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.563 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.563 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.564 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473783+0000", 2023-10-09T06:36:21.564 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.564 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.564 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.564 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.565 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473783+0000", 2023-10-09T06:36:21.565 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.565 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.565 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.565 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473781+0000", 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4279999999999996e-06 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.566 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473789+0000", 2023-10-09T06:36:21.567 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5000999999999999e-05 2023-10-09T06:36:21.567 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.567 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.567 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.567 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473804+0000", 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473811+0000", 2023-10-09T06:36:21.568 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5982000000000004e-05 2023-10-09T06:36:21.569 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.569 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.569 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.569 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.473897+0000", 2023-10-09T06:36:21.569 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046237810000000004 2023-10-09T06:36:21.570 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.570 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.570 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.570 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.478520+0000", 2023-10-09T06:36:21.570 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000199292 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.478720+0000", 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040105460000000002 2023-10-09T06:36:21.571 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.572 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.572 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.572 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.482730+0000", 2023-10-09T06:36:21.572 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0086246139999999992 2023-10-09T06:36:21.572 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.573 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.573 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.573 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.491355+0000", 2023-10-09T06:36:21.573 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.153617853 2023-10-09T06:36:21.573 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.574 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.574 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.574 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.644973+0000", 2023-10-09T06:36:21.574 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1681000000000006e-05 2023-10-09T06:36:21.574 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.645054+0000", 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9251000000000005e-05 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:30:37.645124+0000", 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.171340835 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.575 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:162612 7.7 7:ecabb1a3:::benchmark_data_smithi125_153312_object57925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:31:29.743648+0000", 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 291.70326691499997, 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22622615300000001, 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162612 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743648+0000", 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.577 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743648+0000", 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743647+0000", 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0999999999999998e-06 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743654+0000", 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2136e-05 2023-10-09T06:36:21.578 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743666+0000", 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3479999999999998e-06 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743673+0000", 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0577000000000001e-05 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.579 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743714+0000", 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015172499999999999 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.580 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743866+0000", 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8937999999999994e-05 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.743964+0000", 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010867903999999999 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.754832+0000", 2023-10-09T06:36:21.581 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0031518269999999998 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.757984+0000", 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.211511967 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.969496+0000", 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000272861 2023-10-09T06:36:21.582 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.969769+0000", 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105306 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.969874+0000", 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22622615300000001 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.583 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:162614 7.7 7:e1dfdb9f:::benchmark_data_smithi125_153312_object57927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:31:29.744818+0000", 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 291.70209748000002, 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22566138599999999, 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.584 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162614 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744818+0000", 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.585 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744818+0000", 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744816+0000", 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1319999999999996e-06 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744823+0000", 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6232999999999999e-05 2023-10-09T06:36:21.586 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744840+0000", 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0330000000000003e-06 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.744847+0000", 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042885650000000003 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.587 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.749135+0000", 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012094799999999999 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.749256+0000", 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013556399999999999 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.588 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.749392+0000", 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0083225590000000002 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.757714+0000", 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001655396 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.759370+0000", 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21091938900000001 2023-10-09T06:36:21.589 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.970289+0000", 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000110744 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.970400+0000", 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9080999999999997e-05 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.590 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:29.970479+0000", 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22566138599999999 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:164327 7.7 7:e2211afa:::benchmark_data_smithi125_153312_object59640:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:31:32.761975+0000", 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 288.68494056899999, 2023-10-09T06:36:21.591 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.186887726, 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 164327 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.761975+0000", 2023-10-09T06:36:21.592 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.761975+0000", 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.761973+0000", 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5400000000000001e-06 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.593 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.761982+0000", 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6560000000000001e-05 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.761999+0000", 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.024e-06 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.594 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.762006+0000", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3482999999999997e-05 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.762049+0000", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010563100000000001 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.762155+0000", 2023-10-09T06:36:21.595 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103795 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.762259+0000", 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0052518799999999996 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767511+0000", 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0043409759999999999 2023-10-09T06:36:21.596 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.771852+0000", 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.17690977799999999 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.948761+0000", 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6867000000000003e-05 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.597 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.948818+0000", 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3909999999999998e-05 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.948862+0000", 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.186887726 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.598 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:164329 7.7 7:edb711b4:::benchmark_data_smithi125_153312_object59642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:31:32.767566+0000", 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 288.67934884200002, 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18148747300000001, 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.599 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 164329 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767566+0000", 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767566+0000", 2023-10-09T06:36:21.600 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767564+0000", 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7170000000000003e-06 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767574+0000", 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4073999999999999e-05 2023-10-09T06:36:21.601 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767588+0000", 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0020000000000003e-06 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767594+0000", 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1081000000000002e-05 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.602 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767635+0000", 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014155799999999999 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767777+0000", 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118286 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.603 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.767895+0000", 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0040349590000000003 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.771930+0000", 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8993999999999994e-05 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.604 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.772029+0000", 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.176935811 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.948965+0000", 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9271000000000003e-05 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.949014+0000", 2023-10-09T06:36:21.605 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9597000000000001e-05 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:31:32.949054+0000", 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18148747300000001 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.606 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:191648 7.7 7:e6f16f1e:::benchmark_data_smithi125_153312_object86961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:14.227123+0000", 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 247.21979210699999, 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.23358003399999999, 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191648 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.607 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227123+0000", 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227123+0000", 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.608 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227121+0000", 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7640000000000005e-06 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227129+0000", 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7280000000000001e-05 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227146+0000", 2023-10-09T06:36:21.609 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5429999999999999e-06 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227153+0000", 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9041e-05 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227202+0000", 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014282699999999999 2023-10-09T06:36:21.610 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227345+0000", 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013208999999999999 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.227477+0000", 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.005259896 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.611 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.232737+0000", 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0015096669999999999 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.234246+0000", 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.22633996000000001 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.612 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460586+0000", 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5449000000000003e-05 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460652+0000", 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1209999999999999e-05 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460703+0000", 2023-10-09T06:36:21.613 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.23358003399999999 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:191653 7.7 7:eaf7d2ec:::benchmark_data_smithi125_153312_object86966:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:14.237644+0000", 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 247.20927112000001, 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.223238624, 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.614 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 191653 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237644+0000", 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.615 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237644+0000", 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237642+0000", 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4669999999999993e-06 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.616 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237652+0000", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7731000000000002e-05 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237689+0000", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1532000000000001e-05 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237701+0000", 2023-10-09T06:36:21.617 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5828000000000002e-05 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237747+0000", 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012345600000000001 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237870+0000", 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010873499999999999 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.618 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.237979+0000", 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049095110000000001 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.242888+0000", 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0474000000000001e-05 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.619 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.242979+0000", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21778519599999999 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460764+0000", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4508e-05 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460829+0000", 2023-10-09T06:36:21.620 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3965000000000003e-05 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:14.460883+0000", 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.223238624 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:194998 7.7 7:ea6ded7c:::benchmark_data_smithi125_153312_object90311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.621 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:19.496707+0000", 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 241.95020853599999, 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15109904900000001, 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 194998 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.622 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496707+0000", 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496707+0000", 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.623 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496705+0000", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4530000000000008e-06 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496714+0000", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6115000000000001e-05 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496730+0000", 2023-10-09T06:36:21.624 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3639999999999999e-06 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496739+0000", 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0028000000000003e-05 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496789+0000", 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013657799999999999 2023-10-09T06:36:21.625 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.496925+0000", 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000145859 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.497071+0000", 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049262250000000002 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.626 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.501997+0000", 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012975965000000001 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.514973+0000", 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13268163099999999 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.627 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.647655+0000", 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3028000000000005e-05 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.647738+0000", 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7637000000000003e-05 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:19.647806+0000", 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15109904900000001 2023-10-09T06:36:21.628 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:197393 7.7 7:e974270a:::benchmark_data_smithi125_153312_object92706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:23.174804+0000", 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 238.27211069399999, 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.197037359, 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.629 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 197393 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174804+0000", 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.630 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174804+0000", 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174802+0000", 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0288e-05 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.631 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174813+0000", 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.012e-05 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174833+0000", 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9729999999999997e-06 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174841+0000", 2023-10-09T06:36:21.632 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4870000000000006e-05 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.174926+0000", 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017211400000000001 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.175098+0000", 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017318800000000001 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.633 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.175271+0000", 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006047345 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.181318+0000", 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007592077 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.634 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.188910+0000", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.18270808599999999 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.371618+0000", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000125795 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.371744+0000", 2023-10-09T06:36:21.635 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7590999999999994e-05 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:23.371842+0000", 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.197037359 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:199995 7.7 7:e796cbe6:::benchmark_data_smithi125_153312_object95308:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.636 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:27.441925+0000", 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 234.00499029599999, 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15499158199999999, 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 199995 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.637 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441925+0000", 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441925+0000", 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.638 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441924+0000", 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9139999999999996e-06 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441930+0000", 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2342e-05 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441942+0000", 2023-10-09T06:36:21.639 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9820000000000008e-06 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.441950+0000", 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0037366650000000001 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.445687+0000", 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013723199999999999 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.640 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.445824+0000", 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016860399999999999 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.445993+0000", 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051655609999999999 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.641 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.451158+0000", 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.135012675 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.586171+0000", 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010590908 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.596762+0000", 2023-10-09T06:36:21.642 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4908999999999995e-05 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.596857+0000", 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9799999999999997e-05 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.596916+0000", 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15499158199999999 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.643 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45170.0:200010 7.7 7:efa98aad:::benchmark_data_smithi125_153312_object95323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:32:27.462638+0000", 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 233.984277459, 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15189749999999999, 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45170", 2023-10-09T06:36:21.644 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/2748049253", 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 200010 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462638+0000", 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462638+0000", 2023-10-09T06:36:21.645 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462636+0000", 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3109999999999995e-06 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462645+0000", 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7796e-05 2023-10-09T06:36:21.646 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462663+0000", 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8259999999999993e-06 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462671+0000", 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1035000000000002e-05 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.647 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462722+0000", 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016148900000000001 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.462883+0000", 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018574499999999999 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:36:21.648 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.463069+0000", 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0049259969999999997 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.467995+0000", 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14342073299999999 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.611416+0000", 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0030110050000000002 2023-10-09T06:36:21.649 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.614427+0000", 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2952000000000003e-05 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.614489+0000", 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5674999999999999e-05 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:36:21.650 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:32:27.614535+0000", 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15189749999999999 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:36:21.651 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:36:21.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.652 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:21] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-10-09T06:36:21.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:21.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:21.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:21.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:22.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:22.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:22.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:22.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:22.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:22.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:22.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:22 smithi155 bash[59495]: cluster 2023-10-09T06:36:21.174789+0000 mgr.y (mgr.25723) 949 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:36:22.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:36:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:22.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:22.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:22 smithi125 bash[77299]: cluster 2023-10-09T06:36:21.174789+0000 mgr.y (mgr.25723) 949 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:36:22.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:22 smithi125 bash[80002]: cluster 2023-10-09T06:36:21.174789+0000 mgr.y (mgr.25723) 949 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:36:22.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:22.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:22.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:22.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:22.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:22.934 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:22.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:23.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:23.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:23.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:23.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:23.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:23.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:23.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:23.457 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:23.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:23.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:23.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:23.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:23.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:23.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:24.057 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:24.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:24.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:24.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:24.321 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:24 smithi125 bash[80002]: cluster 2023-10-09T06:36:23.176166+0000 mgr.y (mgr.25723) 950 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:36:24.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:24.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:24.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:24.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:24.586 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:24 smithi155 bash[59495]: cluster 2023-10-09T06:36:23.176166+0000 mgr.y (mgr.25723) 950 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:36:24.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:24.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:24 smithi125 bash[77299]: cluster 2023-10-09T06:36:23.176166+0000 mgr.y (mgr.25723) 950 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-09T06:36:24.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:24.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:24.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:24.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:24.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:24.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:25.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:25.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:25.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:25.160 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:25.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:25.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:25.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:25.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:25.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:25.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:25.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:25.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:25.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:25.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:26.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:26.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:26.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:26.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:26.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:26.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:26.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:26.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:26 smithi125 bash[77299]: cluster 2023-10-09T06:36:25.176802+0000 mgr.y (mgr.25723) 951 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T06:36:26.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:26 smithi125 bash[80002]: cluster 2023-10-09T06:36:25.176802+0000 mgr.y (mgr.25723) 951 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T06:36:26.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:26 smithi155 bash[59495]: cluster 2023-10-09T06:36:25.176802+0000 mgr.y (mgr.25723) 951 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2023-10-09T06:36:26.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:26.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:26.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:26.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:26.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:26.912 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:26.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:26.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:27.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:27.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:27.222 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:27.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:27.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:27.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:27.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:27.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:27.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:27.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:27.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:27.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:27.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:27.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:28.011 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:28.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:28.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:28.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:28.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:28.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:28.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:28.407 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:28.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:28.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:28.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:28.612 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:28 smithi125 bash[77299]: cluster 2023-10-09T06:36:27.177411+0000 mgr.y (mgr.25723) 952 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:36:28.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:28 smithi125 bash[80002]: cluster 2023-10-09T06:36:27.177411+0000 mgr.y (mgr.25723) 952 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:36:28.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:28.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:28 smithi155 bash[59495]: cluster 2023-10-09T06:36:27.177411+0000 mgr.y (mgr.25723) 952 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-09T06:36:28.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:28.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:28.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:28.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:28.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:29.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:29.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:29.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:29.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:29.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:29.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:29.421 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:29.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:29.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:29.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:29.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:29.803 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:29.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:29.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:29.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:29.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:29.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:30.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:30.062 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:29 smithi125 bash[77299]: cluster 2023-10-09T06:36:29.177937+0000 mgr.y (mgr.25723) 953 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:36:30.062 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:29 smithi125 bash[80002]: cluster 2023-10-09T06:36:29.177937+0000 mgr.y (mgr.25723) 953 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:36:30.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:30.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:30.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:30.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:30.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:30.266 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:29 smithi155 bash[59495]: cluster 2023-10-09T06:36:29.177937+0000 mgr.y (mgr.25723) 953 : cluster 0 pgmap v967: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-10-09T06:36:30.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:30.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:30.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:30.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:30.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:30.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:30.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:30.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:30.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:30.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:30.804 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:36:30.851 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:36:30.869 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:36:30.877 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:36:30.904 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:36:30.966 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:36:30.998 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:36:31.041 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:36:31.064 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:31.066 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:36:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:31.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:36:31.111 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.111 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.111 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.111 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.111 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.112 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.112 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.112 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:31.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:31.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:31.216 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[77299]: audit 2023-10-09T06:36:30.707986+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.125:0/1206826105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:31.216 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[80002]: audit 2023-10-09T06:36:30.707986+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.125:0/1206826105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:31.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:31.310 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:30 smithi155 bash[59495]: audit 2023-10-09T06:36:30.707986+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.125:0/1206826105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:31.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:31.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:31.575 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:31.582 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:31] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-10-09T06:36:31.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:31.705 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:31.705 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:31.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:31.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:31.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:31.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:31.986 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:36:31.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.997 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:31 smithi155 bash[59495]: audit 2023-10-09T06:36:30.847698+0000 mgr.y (mgr.25723) 954 : audit 0 from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:31.997 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:31 smithi155 bash[59495]: audit 2023-10-09T06:36:31.125537+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:31.997 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:31 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.178487+0000 mgr.y (mgr.25723) 955 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:32.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:32.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:32.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[77299]: audit 2023-10-09T06:36:30.847698+0000 mgr.y (mgr.25723) 954 : audit 0 from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:32.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[77299]: audit 2023-10-09T06:36:31.125537+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:32.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.178487+0000 mgr.y (mgr.25723) 955 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:32.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[80002]: audit 2023-10-09T06:36:30.847698+0000 mgr.y (mgr.25723) 954 : audit 0 from='client.35583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:32.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[80002]: audit 2023-10-09T06:36:31.125537+0000 mon.a (mon.0) 1151 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:32.142 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:31 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.178487+0000 mgr.y (mgr.25723) 955 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-09T06:36:32.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:32.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:32.288 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:32.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:32.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:32.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:32.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:32.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:32.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:32.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:36:32.787 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:32.815 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:32.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:32.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:32.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:32.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:33.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:33.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.951710+0000 mon.a (mon.0) 1152 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.951740+0000 mon.a (mon.0) 1153 : cluster 1 Cluster is now healthy 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: audit 2023-10-09T06:36:31.974938+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.975083+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.977606+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:32 smithi155 bash[59495]: cluster 2023-10-09T06:36:31.981372+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:33.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:33.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.951710+0000 mon.a (mon.0) 1152 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.951740+0000 mon.a (mon.0) 1153 : cluster 1 Cluster is now healthy 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: audit 2023-10-09T06:36:31.974938+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.975083+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.977606+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[80002]: cluster 2023-10-09T06:36:31.981372+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.951710+0000 mon.a (mon.0) 1152 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:33.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.951740+0000 mon.a (mon.0) 1153 : cluster 1 Cluster is now healthy 2023-10-09T06:36:33.448 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: audit 2023-10-09T06:36:31.974938+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.125:0/2568413761' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:33.449 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.975083+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:36:33.449 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.977606+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.449 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:32 smithi125 bash[77299]: cluster 2023-10-09T06:36:31.981372+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e204 with expected crc 2023-10-09T06:36:33.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:33.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:33.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:33.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:33.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:33.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:33.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:33.983 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[77299]: cluster 2023-10-09T06:36:33.179038+0000 mgr.y (mgr.25723) 956 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:36:33.984 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[77299]: audit 2023-10-09T06:36:33.600499+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:33.984 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[77299]: audit 2023-10-09T06:36:33.601755+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:33.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[80002]: cluster 2023-10-09T06:36:33.179038+0000 mgr.y (mgr.25723) 956 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:36:33.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[80002]: audit 2023-10-09T06:36:33.600499+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:33.984 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:33 smithi125 bash[80002]: audit 2023-10-09T06:36:33.601755+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:33.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:34.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:34.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:34.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:33 smithi155 bash[59495]: cluster 2023-10-09T06:36:33.179038+0000 mgr.y (mgr.25723) 956 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-09T06:36:34.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:33 smithi155 bash[59495]: audit 2023-10-09T06:36:33.600499+0000 mon.a (mon.0) 1156 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:34.134 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:33 smithi155 bash[59495]: audit 2023-10-09T06:36:33.601755+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:34.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:34.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:34.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:34.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:34.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:34.517 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:34.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:34.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:34.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:34.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:34.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:34.856 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":967,"stamp":"2023-10-09T06:36:29.177605+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4296677323,"num_objects":65518,"num_object_clones":0,"num_object_copies":196554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65518,"num_whiteouts":0,"num_read":6290,"num_read_kb":13749,"num_write":153748,"num_write_kb":6995320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":145512,"ondisk_log_size":145512,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16191872,"kb_used_data":12701824,"kb_used_omap":30,"kb_used_meta":3489633,"kb_avail":733769344,"statfs":{"total":767960285184,"available":751379808256,"internally_reserved":0,"allocated":13006667776,"data_stored":13001108740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":3573384303},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,3,1,7,8,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-527171584,"num_objects":-8044,"num_object_clones":0,"num_object_copies":-24132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8044,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004196"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":184,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656738+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:35:45.656738+0000","last_peered":"2023-10-09T06:35:45.656738+0000","last_clean":"2023-10-09T06:35:45.656738+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:35:45.656738+0000","last_undegraded":"2023-10-09T06:35:45.656738+0000","last_fullsized":"2023-10-09T06:35:45.656738+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":179,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652969+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:35:45.652969+0000","last_peered":"2023-10-09T06:35:45.652969+0000","last_clean":"2023-10-09T06:35:45.652969+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:35:45.652969+0000","last_undegraded":"2023-10-09T06:35:45.652969+0000","last_fullsized":"2023-10-09T06:35:45.652969+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.865187+0000","last_change":"2023-10-09T06:35:45.655708+0000","last_active":"2023-10-09T06:35:46.865187+0000","last_peered":"2023-10-09T06:35:46.865187+0000","last_clean":"2023-10-09T06:35:46.865187+0000","last_became_active":"2023-10-09T06:35:45.655323+0000","last_became_peered":"2023-10-09T06:35:45.655323+0000","last_unstale":"2023-10-09T06:35:46.865187+0000","last_undegraded":"2023-10-09T06:35:46.865187+0000","last_fullsized":"2023-10-09T06:35:46.865187+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:00.683127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":188,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.647026+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:35:45.647026+0000","last_peered":"2023-10-09T06:35:45.647026+0000","last_clean":"2023-10-09T06:35:45.647026+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:35:45.647026+0000","last_undegraded":"2023-10-09T06:35:45.647026+0000","last_fullsized":"2023-10-09T06:35:45.647026+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":196,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.279918+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:36:08.279918+0000","last_peered":"2023-10-09T06:36:08.279918+0000","last_clean":"2023-10-09T06:36:08.279918+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:36:08.279918+0000","last_undegraded":"2023-10-09T06:36:08.279918+0000","last_fullsized":"2023-10-09T06:36:08.279918+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":43,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.653737+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:35:45.653737+0000","last_peered":"2023-10-09T06:35:45.653737+0000","last_clean":"2023-10-09T06:35:45.653737+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:35:45.653737+0000","last_undegraded":"2023-10-09T06:35:45.653737+0000","last_fullsized":"2023-10-09T06:35:45.653737+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646758+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:35:45.646758+0000","last_peered":"2023-10-09T06:35:45.646758+0000","last_clean":"2023-10-09T06:35:45.646758+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:35:45.646758+0000","last_undegraded":"2023-10-09T06:35:45.646758+0000","last_fullsized":"2023-10-09T06:35:45.646758+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":211,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652552+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:35:45.652552+0000","last_peered":"2023-10-09T06:35:45.652552+0000","last_clean":"2023-10-09T06:35:45.652552+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:35:45.652552+0000","last_undegraded":"2023-10-09T06:35:45.652552+0000","last_fullsized":"2023-10-09T06:35:45.652552+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":186,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.662166+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:35:45.662166+0000","last_peered":"2023-10-09T06:35:45.662166+0000","last_clean":"2023-10-09T06:35:45.662166+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:35:45.662166+0000","last_undegraded":"2023-10-09T06:35:45.662166+0000","last_fullsized":"2023-10-09T06:35:45.662166+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":206,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.653360+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:35:45.653360+0000","last_peered":"2023-10-09T06:35:45.653360+0000","last_clean":"2023-10-09T06:35:45.653360+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:35:45.653360+0000","last_undegraded":"2023-10-09T06:35:45.653360+0000","last_fullsized":"2023-10-09T06:35:45.653360+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":101,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661789+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:35:45.661789+0000","last_peered":"2023-10-09T06:35:45.661789+0000","last_clean":"2023-10-09T06:35:45.661789+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:35:45.661789+0000","last_undegraded":"2023-10-09T06:35:45.661789+0000","last_fullsized":"2023-10-09T06:35:45.661789+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.664118+0000","last_change":"2023-10-09T06:35:45.663547+0000","last_active":"2023-10-09T06:35:45.664118+0000","last_peered":"2023-10-09T06:35:45.664118+0000","last_clean":"2023-10-09T06:35:45.664118+0000","last_became_active":"2023-10-09T06:35:45.663179+0000","last_became_peered":"2023-10-09T06:35:45.663179+0000","last_unstale":"2023-10-09T06:35:45.664118+0000","last_undegraded":"2023-10-09T06:35:45.664118+0000","last_fullsized":"2023-10-09T06:35:45.664118+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":251,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.658342+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:35:45.658342+0000","last_peered":"2023-10-09T06:35:45.658342+0000","last_clean":"2023-10-09T06:35:45.658342+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:35:45.658342+0000","last_undegraded":"2023-10-09T06:35:45.658342+0000","last_fullsized":"2023-10-09T06:35:45.658342+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.657611+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:35:45.657611+0000","last_peered":"2023-10-09T06:35:45.657611+0000","last_clean":"2023-10-09T06:35:45.657611+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:35:45.657611+0000","last_undegraded":"2023-10-09T06:35:45.657611+0000","last_fullsized":"2023-10-09T06:35:45.657611+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":101,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661399+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:35:45.661399+0000","last_peered":"2023-10-09T06:35:45.661399+0000","last_clean":"2023-10-09T06:35:45.661399+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:35:45.661399+0000","last_undegraded":"2023-10-09T06:35:45.661399+0000","last_fullsized":"2023-10-09T06:35:45.661399+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.866442+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:35:46.866442+0000","last_peered":"2023-10-09T06:35:46.866442+0000","last_clean":"2023-10-09T06:35:46.866442+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:35:46.866442+0000","last_undegraded":"2023-10-09T06:35:46.866442+0000","last_fullsized":"2023-10-09T06:35:46.866442+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646850+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:35:45.646850+0000","last_peered":"2023-10-09T06:35:45.646850+0000","last_clean":"2023-10-09T06:35:45.646850+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:35:45.646850+0000","last_undegraded":"2023-10-09T06:35:45.646850+0000","last_fullsized":"2023-10-09T06:35:45.646850+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.662771+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:35:45.662771+0000","last_peered":"2023-10-09T06:35:45.662771+0000","last_clean":"2023-10-09T06:35:45.662771+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:35:45.662771+0000","last_undegraded":"2023-10-09T06:35:45.662771+0000","last_fullsized":"2023-10-09T06:35:45.662771+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661066+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:35:45.661066+0000","last_peered":"2023-10-09T06:35:45.661066+0000","last_clean":"2023-10-09T06:35:45.661066+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:35:45.661066+0000","last_undegraded":"2023-10-09T06:35:45.661066+0000","last_fullsized":"2023-10-09T06:35:45.661066+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":177,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646930+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:35:45.646930+0000","last_peered":"2023-10-09T06:35:45.646930+0000","last_clean":"2023-10-09T06:35:45.646930+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:35:45.646930+0000","last_undegraded":"2023-10-09T06:35:45.646930+0000","last_fullsized":"2023-10-09T06:35:45.646930+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":53,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652641+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:35:45.652641+0000","last_peered":"2023-10-09T06:35:45.652641+0000","last_clean":"2023-10-09T06:35:45.652641+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:35:45.652641+0000","last_undegraded":"2023-10-09T06:35:45.652641+0000","last_fullsized":"2023-10-09T06:35:45.652641+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":211,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.289503+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:36:08.289503+0000","last_peered":"2023-10-09T06:36:08.289503+0000","last_clean":"2023-10-09T06:36:08.289503+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:36:08.289503+0000","last_undegraded":"2023-10-09T06:36:08.289503+0000","last_fullsized":"2023-10-09T06:36:08.289503+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"203'9444","reported_seq":9463,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.711389+0000","last_change":"2023-10-09T06:33:33.324346+0000","last_active":"2023-10-09T06:36:24.711389+0000","last_peered":"2023-10-09T06:36:24.711389+0000","last_clean":"2023-10-09T06:36:24.711389+0000","last_became_active":"2023-10-09T06:33:33.323046+0000","last_became_peered":"2023-10-09T06:33:33.323046+0000","last_unstale":"2023-10-09T06:36:24.711389+0000","last_undegraded":"2023-10-09T06:36:24.711389+0000","last_fullsized":"2023-10-09T06:36:24.711389+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9444,"log_dups_size":0,"ondisk_log_size":9444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9444,"num_write_kb":436864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":99,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661689+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:35:45.661689+0000","last_peered":"2023-10-09T06:35:45.661689+0000","last_clean":"2023-10-09T06:35:45.661689+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:35:45.661689+0000","last_undegraded":"2023-10-09T06:35:45.661689+0000","last_fullsized":"2023-10-09T06:35:45.661689+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":188,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.663071+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:35:45.663071+0000","last_peered":"2023-10-09T06:35:45.663071+0000","last_clean":"2023-10-09T06:35:45.663071+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:35:45.663071+0000","last_undegraded":"2023-10-09T06:35:45.663071+0000","last_fullsized":"2023-10-09T06:35:45.663071+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"203'9402","reported_seq":9421,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:25.011076+0000","last_change":"2023-10-09T06:33:33.593835+0000","last_active":"2023-10-09T06:36:25.011076+0000","last_peered":"2023-10-09T06:36:25.011076+0000","last_clean":"2023-10-09T06:36:25.011076+0000","last_became_active":"2023-10-09T06:33:33.593363+0000","last_became_peered":"2023-10-09T06:33:33.593363+0000","last_unstale":"2023-10-09T06:36:25.011076+0000","last_undegraded":"2023-10-09T06:36:25.011076+0000","last_fullsized":"2023-10-09T06:36:25.011076+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9402,"log_dups_size":0,"ondisk_log_size":9402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9402,"num_write_kb":433408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":99,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661933+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:35:45.661933+0000","last_peered":"2023-10-09T06:35:45.661933+0000","last_clean":"2023-10-09T06:35:45.661933+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:35:45.661933+0000","last_undegraded":"2023-10-09T06:35:45.661933+0000","last_fullsized":"2023-10-09T06:35:45.661933+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656357+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:35:45.656357+0000","last_peered":"2023-10-09T06:35:45.656357+0000","last_clean":"2023-10-09T06:35:45.656357+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:35:45.656357+0000","last_undegraded":"2023-10-09T06:35:45.656357+0000","last_fullsized":"2023-10-09T06:35:45.656357+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"203'9653","reported_seq":9674,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:28.990165+0000","last_change":"2023-10-09T06:33:33.587458+0000","last_active":"2023-10-09T06:36:28.990165+0000","last_peered":"2023-10-09T06:36:28.990165+0000","last_clean":"2023-10-09T06:36:28.990165+0000","last_became_active":"2023-10-09T06:33:33.586581+0000","last_became_peered":"2023-10-09T06:33:33.586581+0000","last_unstale":"2023-10-09T06:36:28.990165+0000","last_undegraded":"2023-10-09T06:36:28.990165+0000","last_fullsized":"2023-10-09T06:36:28.990165+0000","mapping_epoch":192,"log_start":"196'200","ondisk_log_start":"196'200","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9453,"log_dups_size":0,"ondisk_log_size":9453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.486216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262864896,"num_objects":4011,"num_object_clones":0,"num_object_copies":12033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9653,"num_write_kb":437248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":125,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661447+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:35:45.661447+0000","last_peered":"2023-10-09T06:35:45.661447+0000","last_clean":"2023-10-09T06:35:45.661447+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:35:45.661447+0000","last_undegraded":"2023-10-09T06:35:45.661447+0000","last_fullsized":"2023-10-09T06:35:45.661447+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":43,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652852+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:35:45.652852+0000","last_peered":"2023-10-09T06:35:45.652852+0000","last_clean":"2023-10-09T06:35:45.652852+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:35:45.652852+0000","last_undegraded":"2023-10-09T06:35:45.652852+0000","last_fullsized":"2023-10-09T06:35:45.652852+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"203'9525","reported_seq":9553,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:26.900505+0000","last_change":"2023-10-09T06:35:45.678703+0000","last_active":"2023-10-09T06:36:26.900505+0000","last_peered":"2023-10-09T06:36:26.900505+0000","last_clean":"2023-10-09T06:36:26.900505+0000","last_became_active":"2023-10-09T06:35:45.655393+0000","last_became_peered":"2023-10-09T06:35:45.655393+0000","last_unstale":"2023-10-09T06:36:26.900505+0000","last_undegraded":"2023-10-09T06:36:26.900505+0000","last_fullsized":"2023-10-09T06:36:26.900505+0000","mapping_epoch":201,"log_start":"196'1400","ondisk_log_start":"196'1400","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8125,"log_dups_size":0,"ondisk_log_size":8125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:26.263052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262733824,"num_objects":4009,"num_object_clones":0,"num_object_copies":12027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9525,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.663781+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:35:45.663781+0000","last_peered":"2023-10-09T06:35:45.663781+0000","last_clean":"2023-10-09T06:35:45.663781+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:35:45.663781+0000","last_undegraded":"2023-10-09T06:35:45.663781+0000","last_fullsized":"2023-10-09T06:35:45.663781+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":160,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646552+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:35:45.646552+0000","last_peered":"2023-10-09T06:35:45.646552+0000","last_clean":"2023-10-09T06:35:45.646552+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:35:45.646552+0000","last_undegraded":"2023-10-09T06:35:45.646552+0000","last_fullsized":"2023-10-09T06:35:45.646552+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"203'9665","reported_seq":9686,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:26.902039+0000","last_change":"2023-10-09T06:33:33.320528+0000","last_active":"2023-10-09T06:36:26.902039+0000","last_peered":"2023-10-09T06:36:26.902039+0000","last_clean":"2023-10-09T06:36:26.902039+0000","last_became_active":"2023-10-09T06:33:33.320042+0000","last_became_peered":"2023-10-09T06:33:33.320042+0000","last_unstale":"2023-10-09T06:36:26.902039+0000","last_undegraded":"2023-10-09T06:36:26.902039+0000","last_fullsized":"2023-10-09T06:36:26.902039+0000","mapping_epoch":192,"log_start":"196'1500","ondisk_log_start":"196'1500","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8165,"log_dups_size":0,"ondisk_log_size":8165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.099867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9665,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":179,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652639+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:35:45.652639+0000","last_peered":"2023-10-09T06:35:45.652639+0000","last_clean":"2023-10-09T06:35:45.652639+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:35:45.652639+0000","last_undegraded":"2023-10-09T06:35:45.652639+0000","last_fullsized":"2023-10-09T06:35:45.652639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.658190+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:35:45.658190+0000","last_peered":"2023-10-09T06:35:45.658190+0000","last_clean":"2023-10-09T06:35:45.658190+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:35:45.658190+0000","last_undegraded":"2023-10-09T06:35:45.658190+0000","last_fullsized":"2023-10-09T06:35:45.658190+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"203'9714","reported_seq":9741,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:28.989687+0000","last_change":"2023-10-09T06:35:45.672810+0000","last_active":"2023-10-09T06:36:28.989687+0000","last_peered":"2023-10-09T06:36:28.989687+0000","last_clean":"2023-10-09T06:36:28.989687+0000","last_became_active":"2023-10-09T06:35:45.654041+0000","last_became_peered":"2023-10-09T06:35:45.654041+0000","last_unstale":"2023-10-09T06:36:28.989687+0000","last_undegraded":"2023-10-09T06:36:28.989687+0000","last_fullsized":"2023-10-09T06:36:28.989687+0000","mapping_epoch":201,"log_start":"196'300","ondisk_log_start":"196'300","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9414,"log_dups_size":0,"ondisk_log_size":9414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:59.364217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260046848,"num_objects":3968,"num_object_clones":0,"num_object_copies":11904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9714,"num_write_kb":437824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":133,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656921+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:35:45.656921+0000","last_peered":"2023-10-09T06:35:45.656921+0000","last_clean":"2023-10-09T06:35:45.656921+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:35:45.656921+0000","last_undegraded":"2023-10-09T06:35:45.656921+0000","last_fullsized":"2023-10-09T06:35:45.656921+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.658589+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:35:45.658589+0000","last_peered":"2023-10-09T06:35:45.658589+0000","last_clean":"2023-10-09T06:35:45.658589+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:35:45.658589+0000","last_undegraded":"2023-10-09T06:35:45.658589+0000","last_fullsized":"2023-10-09T06:35:45.658589+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"203'9577","reported_seq":9595,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.690310+0000","last_change":"2023-10-09T06:33:33.244614+0000","last_active":"2023-10-09T06:36:24.690310+0000","last_peered":"2023-10-09T06:36:24.690310+0000","last_clean":"2023-10-09T06:36:24.690310+0000","last_became_active":"2023-10-09T06:33:33.243719+0000","last_became_peered":"2023-10-09T06:33:33.243719+0000","last_unstale":"2023-10-09T06:36:24.690310+0000","last_undegraded":"2023-10-09T06:36:24.690310+0000","last_fullsized":"2023-10-09T06:36:24.690310+0000","mapping_epoch":192,"log_start":"196'1600","ondisk_log_start":"196'1600","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276365312,"num_objects":4217,"num_object_clones":0,"num_object_copies":12651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9577,"num_write_kb":441408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":186,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656187+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:35:45.656187+0000","last_peered":"2023-10-09T06:35:45.656187+0000","last_clean":"2023-10-09T06:35:45.656187+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:35:45.656187+0000","last_undegraded":"2023-10-09T06:35:45.656187+0000","last_fullsized":"2023-10-09T06:35:45.656187+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.657254+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:35:45.657254+0000","last_peered":"2023-10-09T06:35:45.657254+0000","last_clean":"2023-10-09T06:35:45.657254+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:35:45.657254+0000","last_undegraded":"2023-10-09T06:35:45.657254+0000","last_fullsized":"2023-10-09T06:35:45.657254+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"203'9360","reported_seq":9388,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.732905+0000","last_change":"2023-10-09T06:35:45.697874+0000","last_active":"2023-10-09T06:36:24.732905+0000","last_peered":"2023-10-09T06:36:24.732905+0000","last_clean":"2023-10-09T06:36:24.732905+0000","last_became_active":"2023-10-09T06:35:45.668802+0000","last_became_peered":"2023-10-09T06:35:45.668802+0000","last_unstale":"2023-10-09T06:36:24.732905+0000","last_undegraded":"2023-10-09T06:36:24.732905+0000","last_fullsized":"2023-10-09T06:36:24.732905+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9360,"log_dups_size":0,"ondisk_log_size":9360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270663680,"num_objects":4130,"num_object_clones":0,"num_object_copies":12390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9360,"num_write_kb":431680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.279978+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:36:08.279978+0000","last_peered":"2023-10-09T06:36:08.279978+0000","last_clean":"2023-10-09T06:36:08.279978+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:36:08.279978+0000","last_undegraded":"2023-10-09T06:36:08.279978+0000","last_fullsized":"2023-10-09T06:36:08.279978+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.870044+0000","last_change":"2023-10-09T06:35:45.679016+0000","last_active":"2023-10-09T06:35:46.870044+0000","last_peered":"2023-10-09T06:35:46.870044+0000","last_clean":"2023-10-09T06:35:46.870044+0000","last_became_active":"2023-10-09T06:35:45.678002+0000","last_became_peered":"2023-10-09T06:35:45.678002+0000","last_unstale":"2023-10-09T06:35:46.870044+0000","last_undegraded":"2023-10-09T06:35:46.870044+0000","last_fullsized":"2023-10-09T06:35:46.870044+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:22.262853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"203'9616","reported_seq":9637,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:28.942710+0000","last_change":"2023-10-09T06:33:33.317867+0000","last_active":"2023-10-09T06:36:28.942710+0000","last_peered":"2023-10-09T06:36:28.942710+0000","last_clean":"2023-10-09T06:36:28.942710+0000","last_became_active":"2023-10-09T06:33:33.317504+0000","last_became_peered":"2023-10-09T06:33:33.317504+0000","last_unstale":"2023-10-09T06:36:28.942710+0000","last_undegraded":"2023-10-09T06:36:28.942710+0000","last_fullsized":"2023-10-09T06:36:28.942710+0000","mapping_epoch":192,"log_start":"196'200","ondisk_log_start":"196'200","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9416,"log_dups_size":0,"ondisk_log_size":9416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:56.906981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256507904,"num_objects":3914,"num_object_clones":0,"num_object_copies":11742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9616,"num_write_kb":432960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":827,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:48.893275+0000","last_change":"2023-10-09T06:35:45.665207+0000","last_active":"2023-10-09T06:35:48.893275+0000","last_peered":"2023-10-09T06:35:48.893275+0000","last_clean":"2023-10-09T06:35:48.893275+0000","last_became_active":"2023-10-09T06:35:45.664560+0000","last_became_peered":"2023-10-09T06:35:45.664560+0000","last_unstale":"2023-10-09T06:35:48.893275+0000","last_undegraded":"2023-10-09T06:35:48.893275+0000","last_fullsized":"2023-10-09T06:35:48.893275+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:50.880588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.662465+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:35:45.662465+0000","last_peered":"2023-10-09T06:35:45.662465+0000","last_clean":"2023-10-09T06:35:45.662465+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:35:45.662465+0000","last_undegraded":"2023-10-09T06:35:45.662465+0000","last_fullsized":"2023-10-09T06:35:45.662465+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.672917+0000","last_change":"2023-10-09T06:35:45.672632+0000","last_active":"2023-10-09T06:35:45.672917+0000","last_peered":"2023-10-09T06:35:45.672917+0000","last_clean":"2023-10-09T06:35:45.672917+0000","last_became_active":"2023-10-09T06:35:45.672196+0000","last_became_peered":"2023-10-09T06:35:45.672196+0000","last_unstale":"2023-10-09T06:35:45.672917+0000","last_undegraded":"2023-10-09T06:35:45.672917+0000","last_fullsized":"2023-10-09T06:35:45.672917+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"203'9379","reported_seq":9398,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.814103+0000","last_change":"2023-10-09T06:33:33.593915+0000","last_active":"2023-10-09T06:36:24.814103+0000","last_peered":"2023-10-09T06:36:24.814103+0000","last_clean":"2023-10-09T06:36:24.814103+0000","last_became_active":"2023-10-09T06:33:33.593144+0000","last_became_peered":"2023-10-09T06:33:33.593144+0000","last_unstale":"2023-10-09T06:36:24.814103+0000","last_undegraded":"2023-10-09T06:36:24.814103+0000","last_fullsized":"2023-10-09T06:36:24.814103+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9379,"log_dups_size":0,"ondisk_log_size":9379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271646720,"num_objects":4145,"num_object_clones":0,"num_object_copies":12435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9379,"num_write_kb":432768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":841,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.663924+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:35:45.663924+0000","last_peered":"2023-10-09T06:35:45.663924+0000","last_clean":"2023-10-09T06:35:45.663924+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:35:45.663924+0000","last_undegraded":"2023-10-09T06:35:45.663924+0000","last_fullsized":"2023-10-09T06:35:45.663924+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":186,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.657361+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:35:45.657361+0000","last_peered":"2023-10-09T06:35:45.657361+0000","last_clean":"2023-10-09T06:35:45.657361+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:35:45.657361+0000","last_undegraded":"2023-10-09T06:35:45.657361+0000","last_fullsized":"2023-10-09T06:35:45.657361+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.297499+0000","last_change":"2023-10-09T06:35:45.655403+0000","last_active":"2023-10-09T06:35:46.297499+0000","last_peered":"2023-10-09T06:35:46.297499+0000","last_clean":"2023-10-09T06:35:46.297499+0000","last_became_active":"2023-10-09T06:35:45.655106+0000","last_became_peered":"2023-10-09T06:35:45.655106+0000","last_unstale":"2023-10-09T06:35:46.297499+0000","last_undegraded":"2023-10-09T06:35:46.297499+0000","last_fullsized":"2023-10-09T06:35:46.297499+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:20.470509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"203'9484","reported_seq":9505,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:26.901994+0000","last_change":"2023-10-09T06:33:33.319622+0000","last_active":"2023-10-09T06:36:26.901994+0000","last_peered":"2023-10-09T06:36:26.901994+0000","last_clean":"2023-10-09T06:36:26.901994+0000","last_became_active":"2023-10-09T06:33:33.319140+0000","last_became_peered":"2023-10-09T06:33:33.319140+0000","last_unstale":"2023-10-09T06:36:26.901994+0000","last_undegraded":"2023-10-09T06:36:26.901994+0000","last_fullsized":"2023-10-09T06:36:26.901994+0000","mapping_epoch":192,"log_start":"196'1300","ondisk_log_start":"196'1300","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:00.867299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268828672,"num_objects":4102,"num_object_clones":0,"num_object_copies":12306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9484,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":568,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646150+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:35:45.646150+0000","last_peered":"2023-10-09T06:35:45.646150+0000","last_clean":"2023-10-09T06:35:45.646150+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:35:45.646150+0000","last_undegraded":"2023-10-09T06:35:45.646150+0000","last_fullsized":"2023-10-09T06:35:45.646150+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":142,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.297445+0000","last_change":"2023-10-09T06:35:45.655401+0000","last_active":"2023-10-09T06:35:46.297445+0000","last_peered":"2023-10-09T06:35:46.297445+0000","last_clean":"2023-10-09T06:35:46.297445+0000","last_became_active":"2023-10-09T06:35:45.654688+0000","last_became_peered":"2023-10-09T06:35:45.654688+0000","last_unstale":"2023-10-09T06:35:46.297445+0000","last_undegraded":"2023-10-09T06:35:46.297445+0000","last_fullsized":"2023-10-09T06:35:46.297445+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:17.726367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.669591+0000","last_change":"2023-10-09T06:35:45.669132+0000","last_active":"2023-10-09T06:35:45.669591+0000","last_peered":"2023-10-09T06:35:45.669591+0000","last_clean":"2023-10-09T06:35:45.669591+0000","last_became_active":"2023-10-09T06:35:45.668751+0000","last_became_peered":"2023-10-09T06:35:45.668751+0000","last_unstale":"2023-10-09T06:35:45.669591+0000","last_undegraded":"2023-10-09T06:35:45.669591+0000","last_fullsized":"2023-10-09T06:35:45.669591+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"203'9521","reported_seq":9540,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.796041+0000","last_change":"2023-10-09T06:33:33.244731+0000","last_active":"2023-10-09T06:36:24.796041+0000","last_peered":"2023-10-09T06:36:24.796041+0000","last_clean":"2023-10-09T06:36:24.796041+0000","last_became_active":"2023-10-09T06:33:33.243726+0000","last_became_peered":"2023-10-09T06:33:33.243726+0000","last_unstale":"2023-10-09T06:36:24.796041+0000","last_undegraded":"2023-10-09T06:36:24.796041+0000","last_fullsized":"2023-10-09T06:36:24.796041+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9521,"log_dups_size":0,"ondisk_log_size":9521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271908864,"num_objects":4149,"num_object_clones":0,"num_object_copies":12447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9521,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":551,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.662919+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:35:45.662919+0000","last_peered":"2023-10-09T06:35:45.662919+0000","last_clean":"2023-10-09T06:35:45.662919+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:35:45.662919+0000","last_undegraded":"2023-10-09T06:35:45.662919+0000","last_fullsized":"2023-10-09T06:35:45.662919+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":445,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661380+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:35:45.661380+0000","last_peered":"2023-10-09T06:35:45.661380+0000","last_clean":"2023-10-09T06:35:45.661380+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:35:45.661380+0000","last_undegraded":"2023-10-09T06:35:45.661380+0000","last_fullsized":"2023-10-09T06:35:45.661380+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652112+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:35:45.652112+0000","last_peered":"2023-10-09T06:35:45.652112+0000","last_clean":"2023-10-09T06:35:45.652112+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:35:45.652112+0000","last_undegraded":"2023-10-09T06:35:45.652112+0000","last_fullsized":"2023-10-09T06:35:45.652112+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":158,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.657972+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:35:45.657972+0000","last_peered":"2023-10-09T06:35:45.657972+0000","last_clean":"2023-10-09T06:35:45.657972+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:35:45.657972+0000","last_undegraded":"2023-10-09T06:35:45.657972+0000","last_fullsized":"2023-10-09T06:35:45.657972+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"203'9580","reported_seq":9613,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:26.344978+0000","last_change":"2023-10-09T06:35:45.735978+0000","last_active":"2023-10-09T06:36:26.344978+0000","last_peered":"2023-10-09T06:36:26.344978+0000","last_clean":"2023-10-09T06:36:26.344978+0000","last_became_active":"2023-10-09T06:35:45.673168+0000","last_became_peered":"2023-10-09T06:35:45.673168+0000","last_unstale":"2023-10-09T06:36:26.344978+0000","last_undegraded":"2023-10-09T06:36:26.344978+0000","last_fullsized":"2023-10-09T06:36:26.344978+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9580,"log_dups_size":0,"ondisk_log_size":9580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:06.690354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271843328,"num_objects":4148,"num_object_clones":0,"num_object_copies":12444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9580,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":124,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661204+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:35:45.661204+0000","last_peered":"2023-10-09T06:35:45.661204+0000","last_clean":"2023-10-09T06:35:45.661204+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:35:45.661204+0000","last_undegraded":"2023-10-09T06:35:45.661204+0000","last_fullsized":"2023-10-09T06:35:45.661204+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.274350+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:36:08.274350+0000","last_peered":"2023-10-09T06:36:08.274350+0000","last_clean":"2023-10-09T06:36:08.274350+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:36:08.274350+0000","last_undegraded":"2023-10-09T06:36:08.274350+0000","last_fullsized":"2023-10-09T06:36:08.274350+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.647246+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:35:45.647246+0000","last_peered":"2023-10-09T06:35:45.647246+0000","last_clean":"2023-10-09T06:35:45.647246+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:35:45.647246+0000","last_undegraded":"2023-10-09T06:35:45.647246+0000","last_fullsized":"2023-10-09T06:35:45.647246+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"203'9418","reported_seq":9439,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:26.942469+0000","last_change":"2023-10-09T06:33:33.320380+0000","last_active":"2023-10-09T06:36:26.942469+0000","last_peered":"2023-10-09T06:36:26.942469+0000","last_clean":"2023-10-09T06:36:26.942469+0000","last_became_active":"2023-10-09T06:33:33.319960+0000","last_became_peered":"2023-10-09T06:33:33.319960+0000","last_unstale":"2023-10-09T06:36:26.942469+0000","last_undegraded":"2023-10-09T06:36:26.942469+0000","last_fullsized":"2023-10-09T06:36:26.942469+0000","mapping_epoch":192,"log_start":"196'1300","ondisk_log_start":"196'1300","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:25.704605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264241152,"num_objects":4032,"num_object_clones":0,"num_object_copies":12096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9418,"num_write_kb":430400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":113,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.661447+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:35:45.661447+0000","last_peered":"2023-10-09T06:35:45.661447+0000","last_clean":"2023-10-09T06:35:45.661447+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:35:45.661447+0000","last_undegraded":"2023-10-09T06:35:45.661447+0000","last_fullsized":"2023-10-09T06:35:45.661447+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:46.297516+0000","last_change":"2023-10-09T06:35:45.662792+0000","last_active":"2023-10-09T06:35:46.297516+0000","last_peered":"2023-10-09T06:35:46.297516+0000","last_clean":"2023-10-09T06:35:46.297516+0000","last_became_active":"2023-10-09T06:35:45.662459+0000","last_became_peered":"2023-10-09T06:35:45.662459+0000","last_unstale":"2023-10-09T06:35:46.297516+0000","last_undegraded":"2023-10-09T06:35:46.297516+0000","last_fullsized":"2023-10-09T06:35:46.297516+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:05.231805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":173,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.274796+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:36:08.274796+0000","last_peered":"2023-10-09T06:36:08.274796+0000","last_clean":"2023-10-09T06:36:08.274796+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:36:08.274796+0000","last_undegraded":"2023-10-09T06:36:08.274796+0000","last_fullsized":"2023-10-09T06:36:08.274796+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"203'9947","reported_seq":9968,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:28.986629+0000","last_change":"2023-10-09T06:33:33.586554+0000","last_active":"2023-10-09T06:36:28.986629+0000","last_peered":"2023-10-09T06:36:28.986629+0000","last_clean":"2023-10-09T06:36:28.986629+0000","last_became_active":"2023-10-09T06:33:33.586066+0000","last_became_peered":"2023-10-09T06:33:33.586066+0000","last_unstale":"2023-10-09T06:36:28.986629+0000","last_undegraded":"2023-10-09T06:36:28.986629+0000","last_fullsized":"2023-10-09T06:36:28.986629+0000","mapping_epoch":192,"log_start":"196'500","ondisk_log_start":"196'500","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9447,"log_dups_size":0,"ondisk_log_size":9447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.973752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9947,"num_write_kb":447424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":206,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.289428+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:36:08.289428+0000","last_peered":"2023-10-09T06:36:08.289428+0000","last_clean":"2023-10-09T06:36:08.289428+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:36:08.289428+0000","last_undegraded":"2023-10-09T06:36:08.289428+0000","last_fullsized":"2023-10-09T06:36:08.289428+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":387,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.669358+0000","last_change":"2023-10-09T06:35:45.668825+0000","last_active":"2023-10-09T06:35:45.669358+0000","last_peered":"2023-10-09T06:35:45.669358+0000","last_clean":"2023-10-09T06:35:45.669358+0000","last_became_active":"2023-10-09T06:35:45.668234+0000","last_became_peered":"2023-10-09T06:35:45.668234+0000","last_unstale":"2023-10-09T06:35:45.669358+0000","last_undegraded":"2023-10-09T06:35:45.669358+0000","last_fullsized":"2023-10-09T06:35:45.669358+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":204,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652304+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:35:45.652304+0000","last_peered":"2023-10-09T06:35:45.652304+0000","last_clean":"2023-10-09T06:35:45.652304+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:35:45.652304+0000","last_undegraded":"2023-10-09T06:35:45.652304+0000","last_fullsized":"2023-10-09T06:35:45.652304+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"203'9357","reported_seq":9375,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:24.752877+0000","last_change":"2023-10-09T06:33:33.594495+0000","last_active":"2023-10-09T06:36:24.752877+0000","last_peered":"2023-10-09T06:36:24.752877+0000","last_clean":"2023-10-09T06:36:24.752877+0000","last_became_active":"2023-10-09T06:33:33.593206+0000","last_became_peered":"2023-10-09T06:33:33.593206+0000","last_unstale":"2023-10-09T06:36:24.752877+0000","last_undegraded":"2023-10-09T06:36:24.752877+0000","last_fullsized":"2023-10-09T06:36:24.752877+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9357,"log_dups_size":0,"ondisk_log_size":9357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270401560,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9357,"num_write_kb":431425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":723,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.654153+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:35:45.654153+0000","last_peered":"2023-10-09T06:35:45.654153+0000","last_clean":"2023-10-09T06:35:45.654153+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:35:45.654153+0000","last_undegraded":"2023-10-09T06:35:45.654153+0000","last_fullsized":"2023-10-09T06:35:45.654153+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.647159+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:35:45.647159+0000","last_peered":"2023-10-09T06:35:45.647159+0000","last_clean":"2023-10-09T06:35:45.647159+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:35:45.647159+0000","last_undegraded":"2023-10-09T06:35:45.647159+0000","last_fullsized":"2023-10-09T06:35:45.647159+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656163+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:35:45.656163+0000","last_peered":"2023-10-09T06:35:45.656163+0000","last_clean":"2023-10-09T06:35:45.656163+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:35:45.656163+0000","last_undegraded":"2023-10-09T06:35:45.656163+0000","last_fullsized":"2023-10-09T06:35:45.656163+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":166,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.646706+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:35:45.646706+0000","last_peered":"2023-10-09T06:35:45.646706+0000","last_clean":"2023-10-09T06:35:45.646706+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:35:45.646706+0000","last_undegraded":"2023-10-09T06:35:45.646706+0000","last_fullsized":"2023-10-09T06:35:45.646706+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.647791+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:35:45.647791+0000","last_peered":"2023-10-09T06:35:45.647791+0000","last_clean":"2023-10-09T06:35:45.647791+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:35:45.647791+0000","last_undegraded":"2023-10-09T06:35:45.647791+0000","last_fullsized":"2023-10-09T06:35:45.647791+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":170,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.663916+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:35:45.663916+0000","last_peered":"2023-10-09T06:35:45.663916+0000","last_clean":"2023-10-09T06:35:45.663916+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:35:45.663916+0000","last_undegraded":"2023-10-09T06:35:45.663916+0000","last_fullsized":"2023-10-09T06:35:45.663916+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.657619+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:35:45.657619+0000","last_peered":"2023-10-09T06:35:45.657619+0000","last_clean":"2023-10-09T06:35:45.657619+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:35:45.657619+0000","last_undegraded":"2023-10-09T06:35:45.657619+0000","last_fullsized":"2023-10-09T06:35:45.657619+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":181,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.662620+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:35:45.662620+0000","last_peered":"2023-10-09T06:35:45.662620+0000","last_clean":"2023-10-09T06:35:45.662620+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:35:45.662620+0000","last_undegraded":"2023-10-09T06:35:45.662620+0000","last_fullsized":"2023-10-09T06:35:45.662620+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656999+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:35:45.656999+0000","last_peered":"2023-10-09T06:35:45.656999+0000","last_clean":"2023-10-09T06:35:45.656999+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:35:45.656999+0000","last_undegraded":"2023-10-09T06:35:45.656999+0000","last_fullsized":"2023-10-09T06:35:45.656999+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":205,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652725+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:35:45.652725+0000","last_peered":"2023-10-09T06:35:45.652725+0000","last_clean":"2023-10-09T06:35:45.652725+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:35:45.652725+0000","last_undegraded":"2023-10-09T06:35:45.652725+0000","last_fullsized":"2023-10-09T06:35:45.652725+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.652376+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:35:45.652376+0000","last_peered":"2023-10-09T06:35:45.652376+0000","last_clean":"2023-10-09T06:35:45.652376+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:35:45.652376+0000","last_undegraded":"2023-10-09T06:35:45.652376+0000","last_fullsized":"2023-10-09T06:35:45.652376+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":165,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:36:08.278582+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:36:08.278582+0000","last_peered":"2023-10-09T06:36:08.278582+0000","last_clean":"2023-10-09T06:36:08.278582+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:36:08.278582+0000","last_undegraded":"2023-10-09T06:36:08.278582+0000","last_fullsized":"2023-10-09T06:36:08.278582+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:35:45.656158+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:35:45.656158+0000","last_peered":"2023-10-09T06:35:45.656158+0000","last_clean":"2023-10-09T06:35:45.656158+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:35:45.656158+0000","last_undegraded":"2023-10-09T06:35:45.656158+0000","last_fullsized":"2023-10-09T06:35:45.656158+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4291887128,"num_objects":65490,"num_object_clones":0,"num_object_copies":196470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65490,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152642,"num_write_kb":6980161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12913487872,"data_stored":12913475656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144342,"ondisk_log_size":144342,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430661,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637912,"kb_used_data":1336592,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92107240,"statfs":{"total":95995035648,"available":94317813760,"internally_reserved":0,"allocated":1368670208,"data_stored":1368106553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:58 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":1.1830000000000001,"15min":2.899},"min":{"1min":0.371,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.4699999999999998,"5min":5.4699999999999998,"15min":178.738},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.153,"5min":1.1579999999999999,"15min":2.798},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":4.391,"5min":5.0380000000000003,"15min":177.75},"last":0.33400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":2.6219999999999999,"15min":2.5009999999999999},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.247},"max":{"1min":5.6260000000000003,"5min":148.88499999999999,"15min":148.88499999999999},"last":0.372},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.6389999999999998,"15min":2.3580000000000001},"min":{"1min":0.29399999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.2069999999999999,"5min":148.059,"15min":148.059},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":1.216,"15min":2.258},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29399999999999998},"max":{"1min":11.006,"5min":11.006,"15min":104.992},"last":2.6760000000000002},{"interface":"front","average":{"1min":1.631,"5min":1.2969999999999999,"15min":2.1179999999999999},"min":{"1min":0.38200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.6269999999999998,"5min":7.0629999999999997,"15min":104.691},"last":2.5950000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.6220000000000001,"15min":3.6150000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":7.3209999999999997,"5min":14.632,"15min":183.971},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.448,"15min":3.363},"min":{"1min":0.376,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.3019999999999996,"5min":14.702,"15min":183.095},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:36:02 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.1659999999999999,"15min":1.1719999999999999},"min":{"1min":0.34100000000000003,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":12.382999999999999,"5min":12.382999999999999,"15min":12.382999999999999},"last":3.9780000000000002},{"interface":"front","average":{"1min":3.4470000000000001,"5min":1.569,"15min":1.6619999999999999},"min":{"1min":0.44700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":16.064,"5min":16.064,"15min":16.064},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:36:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.125,"15min":1.26},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.3230000000000004,"5min":7.1550000000000002,"15min":8.6129999999999995},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.2519999999999998,"5min":1.492,"15min":1.4099999999999999},"min":{"1min":0.435,"5min":0.33900000000000002,"15min":0.33100000000000002},"max":{"1min":6.4610000000000003,"5min":9.0329999999999995,"15min":13.634},"last":4.0229999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:36:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":1.413,"15min":1.2989999999999999},"min":{"1min":0.372,"5min":0.314,"15min":0.314},"max":{"1min":8.2439999999999998,"5min":8.2439999999999998,"15min":15.521000000000001},"last":3.9319999999999999},{"interface":"front","average":{"1min":3.0910000000000002,"5min":1.635,"15min":1.5860000000000001},"min":{"1min":0.42299999999999999,"5min":0.40100000000000002,"15min":0.33400000000000002},"max":{"1min":16.024000000000001,"5min":16.024000000000001,"15min":16.024000000000001},"last":0.61899999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626893,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1641348,"kb_used_data":1335932,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92103804,"statfs":{"total":95995035648,"available":94314295296,"internally_reserved":0,"allocated":1367994368,"data_stored":1367450412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.3560000000000001,"15min":1.2609999999999999},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.255},"max":{"1min":5.9500000000000002,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.91000000000000003},{"interface":"front","average":{"1min":2.347,"5min":1.4570000000000001,"15min":1.4319999999999999},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.26300000000000001},"max":{"1min":10.696,"5min":10.696,"15min":10.696},"last":2.25}]},{"osd":1,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0009999999999999,"5min":4.1260000000000003,"15min":2.7690000000000001},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":19.25,"5min":316.88099999999997,"15min":316.88099999999997},"last":4.8769999999999998},{"interface":"front","average":{"1min":2.363,"5min":4.1719999999999997,"15min":2.8010000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":7.952,"5min":316.84399999999999,"15min":316.84399999999999},"last":0.998}]},{"osd":2,"last update":"Mon Oct 9 06:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":1.617,"15min":2.2810000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":6.2439999999999998,"5min":26.477,"15min":95.540000000000006},"last":0.94499999999999995},{"interface":"front","average":{"1min":2.274,"5min":1.843,"15min":2.2690000000000001},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":7.3390000000000004,"5min":25.745999999999999,"15min":96.352000000000004},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:36:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.1699999999999999,"15min":2.423},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":6.0049999999999999,"5min":27.491,"15min":115.09399999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.728,"5min":1.9630000000000001,"15min":2.153},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.32000000000000001},"max":{"1min":6.6079999999999997,"5min":26.815000000000001,"15min":112.886},"last":4.9059999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.083,"15min":1.2809999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.26000000000000001},"max":{"1min":10.653,"5min":10.653,"15min":10.653},"last":0.505},{"interface":"front","average":{"1min":2.254,"5min":1.266,"15min":1.4730000000000001},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.29799999999999999},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":4.8369999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":1.0760000000000001,"15min":1.458},"min":{"1min":0.40000000000000002,"5min":0.377,"15min":0.34300000000000003},"max":{"1min":10.601000000000001,"5min":10.601000000000001,"15min":11.754},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.6019999999999999,"5min":1.508,"15min":1.623},"min":{"1min":0.45600000000000002,"5min":0.41899999999999998,"15min":0.308},"max":{"1min":8.7219999999999995,"5min":8.7219999999999995,"15min":12.295999999999999},"last":5.5300000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:04 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.2090000000000001,"15min":1.466},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34000000000000002},"max":{"1min":10.673,"5min":10.673,"15min":11.076000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.266,"5min":1.1599999999999999,"15min":1.3560000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.077,"5min":9.4179999999999993,"15min":11.102},"last":1.0209999999999999}]}]},{"osd":1,"up_from":93,"seq":399431958803,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2231000,"kb_used_data":1827984,"kb_used_omap":7,"kb_used_meta":402936,"kb_avail":91514152,"statfs":{"total":95995035648,"available":93710491648,"internally_reserved":0,"allocated":1871855616,"data_stored":1871002347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":412606864},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":3.7850000000000001,"15min":2.5499999999999998},"min":{"1min":0.30299999999999999,"5min":0.216,"15min":0.187},"max":{"1min":8.125,"5min":132.886,"15min":132.886},"last":5.7770000000000001},{"interface":"front","average":{"1min":3.2010000000000001,"5min":4.5250000000000004,"15min":2.8780000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":13.411,"5min":133.315,"15min":133.315},"last":0.23400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":2.117,"15min":1.9430000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.19800000000000001},"max":{"1min":6.4160000000000004,"5min":34.982999999999997,"15min":67.206000000000003},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.55,"5min":2.1920000000000002,"15min":1.988},"min":{"1min":0.32800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":6.1699999999999999,"5min":35.030999999999999,"15min":67.132000000000005},"last":0.35299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":7.258,"5min":4.3449999999999998,"15min":4.516},"min":{"1min":0.27800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":110.711,"5min":132.792,"15min":132.792},"last":0.38600000000000001},{"interface":"front","average":{"1min":8.1660000000000004,"5min":4.3840000000000003,"15min":4.5519999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":111.48999999999999,"5min":132.53299999999999,"15min":132.53299999999999},"last":11.541}]},{"osd":4,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.5049999999999999,"15min":1.5149999999999999},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":6.2549999999999999,"5min":26.068999999999999,"15min":26.068999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5720000000000001,"5min":1.2849999999999999,"15min":1.4810000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.31},"max":{"1min":8.2159999999999993,"5min":8.2159999999999993,"15min":12.907999999999999},"last":5.8170000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.0270000000000001,"15min":1.7230000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.2300000000000004,"5min":10.586,"15min":14.170999999999999},"last":3.6779999999999999},{"interface":"front","average":{"1min":1.9339999999999999,"5min":1.639,"15min":1.6379999999999999},"min":{"1min":0.437,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.9219999999999997,"5min":26.157,"15min":26.157},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":1.9850000000000001,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.2800000000000002,"5min":10.625,"15min":14.212999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.02,"5min":1.8919999999999999,"15min":1.7310000000000001},"min":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.7309999999999999,"5min":26.132000000000001,"15min":26.132000000000001},"last":0.433}]},{"osd":7,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.901,"15min":1.9410000000000001},"min":{"1min":0.47499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.657,"5min":26.109000000000002,"15min":30.774999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.016,"5min":1.4379999999999999,"15min":1.663},"min":{"1min":0.51700000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":10.025,"5min":10.025,"15min":30.815000000000001},"last":5.843}]}]},{"osd":0,"up_from":88,"seq":377957122329,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2079996,"kb_used_data":1546612,"kb_used_omap":8,"kb_used_meta":533367,"kb_avail":91665156,"statfs":{"total":95995035648,"available":93865119744,"internally_reserved":0,"allocated":1583730688,"data_stored":1582902310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":546168064},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":2.4140000000000001,"15min":2.677},"min":{"1min":0.21199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":6.8920000000000003,"5min":25.084,"15min":138.31700000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.153,"15min":2.5059999999999998},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.20200000000000001},"max":{"1min":6.9500000000000002,"5min":25.036999999999999,"15min":144.37},"last":0.38700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:35:47 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":2.1619999999999999,"15min":1.8939999999999999},"min":{"1min":0.22600000000000001,"5min":0.185,"15min":0.185},"max":{"1min":6.3330000000000002,"5min":13.425000000000001,"15min":28.745000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.8240000000000001,"15min":1.77},"min":{"1min":0.22700000000000001,"5min":0.22600000000000001,"15min":0.18099999999999999},"max":{"1min":6.5860000000000003,"5min":16.129000000000001,"15min":28.146999999999998},"last":0.248}]},{"osd":3,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":2.3599999999999999,"15min":2.6219999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":5.3399999999999999,"5min":33.741999999999997,"15min":59.643000000000001},"last":4.9180000000000001},{"interface":"front","average":{"1min":1.494,"5min":2.2999999999999998,"15min":2.4929999999999999},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":5.4050000000000002,"5min":32.859000000000002,"15min":59.762999999999998},"last":0.67000000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.5069999999999999,"15min":1.4890000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":5.3849999999999998,"5min":8.9920000000000009,"15min":8.9920000000000009},"last":0.372},{"interface":"front","average":{"1min":1.244,"5min":1.8180000000000001,"15min":1.7749999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.7309999999999999,"5min":9.2319999999999993,"15min":28.629000000000001},"last":0.88500000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4849999999999999,"5min":2.3439999999999999,"15min":2.0720000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.2800000000000002,"5min":12.356999999999999,"15min":28.574000000000002},"last":0.91500000000000004},{"interface":"front","average":{"1min":2.016,"5min":1.7290000000000001,"15min":1.859},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.32600000000000001},"max":{"1min":5.4720000000000004,"5min":7.6420000000000003,"15min":28.061},"last":0.35799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":1.8120000000000001,"15min":1.7330000000000001},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.5800000000000001,"5min":16.181999999999999,"15min":27.966999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.5620000000000001,"15min":1.466},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.33700000000000002},"max":{"1min":5.4169999999999998,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":0.41499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":1.875,"15min":1.901},"min":{"1min":0.432,"5min":0.40300000000000002,"15min":0.38200000000000001},"max":{"1min":9.5280000000000005,"5min":9.5280000000000005,"15min":28.109999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.5720000000000001,"5min":1.73,"15min":1.833},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.38400000000000001},"max":{"1min":5.6150000000000002,"5min":8.9190000000000005,"15min":28.074999999999999},"last":0.61299999999999999}]}]},{"osd":2,"up_from":98,"seq":420906795277,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2094720,"kb_used_data":1577144,"kb_used_omap":11,"kb_used_meta":517492,"kb_avail":91650432,"statfs":{"total":95995035648,"available":93850042368,"internally_reserved":0,"allocated":1614995456,"data_stored":1614160298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":529912766},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":2.8740000000000001,"15min":2.3940000000000001},"min":{"1min":0.21299999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":8.9649999999999999,"5min":64.356999999999999,"15min":117.2},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.246,"5min":2.0800000000000001,"15min":2.306},"min":{"1min":0.32800000000000001,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":3.6309999999999998,"5min":63.968000000000004,"15min":121.521},"last":2.153}]},{"osd":1,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":5.6559999999999997,"5min":3.468,"15min":2.282},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.17999999999999999},"max":{"1min":57.802,"5min":60.155000000000001,"15min":60.155000000000001},"last":1.9710000000000001},{"interface":"front","average":{"1min":4.8109999999999999,"5min":2.9780000000000002,"15min":2.2709999999999999},"min":{"1min":0.36599999999999999,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":53.771999999999998,"5min":62.582999999999998,"15min":62.582999999999998},"last":0.90900000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.4700000000000002,"15min":3.5640000000000001},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":9.9670000000000005,"5min":27.050999999999998,"15min":201.33600000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":3.7519999999999998,"5min":2.2269999999999999,"15min":3.6070000000000002},"min":{"1min":0.32600000000000001,"5min":0.28399999999999997,"15min":0.21299999999999999},"max":{"1min":32.167000000000002,"5min":32.167000000000002,"15min":199.95099999999999},"last":8.2050000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.417,"15min":1.8799999999999999},"min":{"1min":0.42999999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.1630000000000003,"5min":5.7439999999999998,"15min":19.184000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.036,"5min":1.2350000000000001,"15min":1.645},"min":{"1min":0.36299999999999999,"5min":0.35199999999999998,"15min":0.33800000000000002},"max":{"1min":4.2279999999999998,"5min":6.3049999999999997,"15min":29.408000000000001},"last":0.35099999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.395,"15min":2.2589999999999999},"min":{"1min":0.377,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":5.468,"5min":5.7080000000000002,"15min":86.423000000000002},"last":0.88500000000000001},{"interface":"front","average":{"1min":2.1960000000000002,"5min":1.621,"15min":2.0779999999999998},"min":{"1min":0.42699999999999999,"5min":0.377,"15min":0.33800000000000002},"max":{"1min":6.0529999999999999,"5min":12.532999999999999,"15min":86.382999999999996},"last":0.38300000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.512,"15min":1.657},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.32800000000000001},"max":{"1min":4.2039999999999997,"5min":8.2959999999999994,"15min":29.504000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.837,"5min":1.7709999999999999,"15min":1.6899999999999999},"min":{"1min":0.40200000000000002,"5min":0.35499999999999998,"15min":0.33800000000000002},"max":{"1min":9.0440000000000005,"5min":9.0440000000000005,"15min":9.0440000000000005},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":1.915,"15min":1.8120000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.38700000000000001},"max":{"1min":9.9209999999999994,"5min":9.9209999999999994,"15min":9.9209999999999994},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.157,"5min":1.627,"15min":1.9330000000000001},"min":{"1min":0.56299999999999994,"5min":0.377,"15min":0.35399999999999998},"max":{"1min":6.5730000000000004,"5min":6.5730000000000004,"15min":12.051},"last":0.85099999999999998}]}]},{"osd":3,"up_from":103,"seq":442381631751,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2104392,"kb_used_data":1578496,"kb_used_omap":3,"kb_used_meta":525820,"kb_avail":91640760,"statfs":{"total":95995035648,"available":93840138240,"internally_reserved":0,"allocated":1616379904,"data_stored":1615537753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":538440225},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.1059999999999999,"15min":2.52},"min":{"1min":0.26200000000000001,"5min":0.224,"15min":0.17399999999999999},"max":{"1min":6.7720000000000002,"5min":27.911999999999999,"15min":147.988},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":2.2469999999999999,"15min":2.375},"min":{"1min":0.22800000000000001,"5min":0.19,"15min":0.18099999999999999},"max":{"1min":11.661,"5min":26.988,"15min":147.94200000000001},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.319,"15min":1.865},"min":{"1min":0.219,"5min":0.20799999999999999,"15min":0.19},"max":{"1min":6.9829999999999997,"5min":47.057000000000002,"15min":47.057000000000002},"last":1.48},{"interface":"front","average":{"1min":1.613,"5min":2.2650000000000001,"15min":2.1379999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.218},"max":{"1min":4.9740000000000002,"5min":54.752000000000002,"15min":54.752000000000002},"last":1.534}]},{"osd":2,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.8280000000000001,"15min":2.2709999999999999},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.20599999999999999},"max":{"1min":7.4279999999999999,"5min":14.968,"15min":47.097999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.573,"5min":1.6040000000000001,"15min":1.7969999999999999},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":6.1109999999999998,"5min":11.683999999999999,"15min":46.997},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":1.728,"15min":1.9870000000000001},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.30399999999999999},"max":{"1min":7.3620000000000001,"5min":8.6470000000000002,"15min":72.653000000000006},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.764,"5min":1.542,"15min":2.0230000000000001},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.048,"5min":11.756,"15min":72.706999999999994},"last":0.71599999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.579,"15min":1.7290000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.9569999999999999,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":2.637,"5min":1.831,"15min":1.867},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":10.315,"5min":10.315,"15min":11.454000000000001},"last":0.68500000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.49,"15min":1.7669999999999999},"min":{"1min":0.42099999999999999,"5min":0.40400000000000003,"15min":0.36099999999999999},"max":{"1min":7.984,"5min":10.384,"15min":15.122999999999999},"last":1.492},{"interface":"front","average":{"1min":1.768,"5min":1.6140000000000001,"15min":1.681},"min":{"1min":0.42299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.0220000000000002,"5min":11.728999999999999,"15min":11.728999999999999},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.1589999999999998,"15min":2.0070000000000001},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":7.6470000000000002,"5min":8.6059999999999999,"15min":18.695},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.6040000000000001,"15min":1.9590000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.39900000000000002},"max":{"1min":4.9539999999999997,"5min":10.409000000000001,"15min":13.211},"last":0.55100000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019359,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1642188,"kb_used_data":1340868,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92102964,"statfs":{"total":95995035648,"available":94313435136,"internally_reserved":0,"allocated":1373048832,"data_stored":1372496827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1030000000000002,"5min":1.8089999999999999,"15min":2.3399999999999999},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.254},"max":{"1min":6.335,"5min":16.218,"15min":115.59699999999999},"last":0.249},{"interface":"front","average":{"1min":2.5209999999999999,"5min":1.8759999999999999,"15min":2.6749999999999998},"min":{"1min":0.39800000000000002,"5min":0.308,"15min":0.27700000000000002},"max":{"1min":6.351,"5min":15.242000000000001,"15min":115.65000000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.6319999999999999,"15min":2.1269999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":3.3439999999999999,"5min":17.033000000000001,"15min":179.47300000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.714,"15min":2.4100000000000001},"min":{"1min":0.377,"5min":0.316,"15min":0.28499999999999998},"max":{"1min":3.3319999999999999,"5min":17.015999999999998,"15min":179.447},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 06:36:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":1.907,"15min":2.016},"min":{"1min":0.443,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":9.0109999999999992,"5min":14.705,"15min":61.783999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.962,"5min":1.659,"15min":2.0449999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.0910000000000002,"5min":6.915,"15min":61.551000000000002},"last":0.56899999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.7090000000000001,"15min":2.1659999999999999},"min":{"1min":0.40899999999999997,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":2.9750000000000001,"5min":8.1449999999999996,"15min":91.361999999999995},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.6679999999999999,"15min":1.8899999999999999},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":2.9620000000000002,"5min":16.257000000000001,"15min":89.676000000000002},"last":3.2519999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.1319999999999999,"15min":1.216},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.1059999999999999,"5min":11.737,"15min":12.242000000000001},"last":6.1500000000000004},{"interface":"front","average":{"1min":3.2839999999999998,"5min":1.7150000000000001,"15min":1.6539999999999999},"min":{"1min":0.47599999999999998,"5min":0.38300000000000001,"15min":0.26500000000000001},"max":{"1min":13.095000000000001,"5min":13.095000000000001,"15min":15.273},"last":0.64800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.548,"15min":1.5449999999999999},"min":{"1min":0.48799999999999999,"5min":0.35399999999999998,"15min":0.314},"max":{"1min":7.9020000000000001,"5min":7.9020000000000001,"15min":18.010000000000002},"last":3.6829999999999998},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.3460000000000001,"15min":1.323},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2880000000000003,"5min":11.782,"15min":11.782},"last":3.2970000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":1.4370000000000001,"15min":1.6830000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.371},"max":{"1min":6.9089999999999998,"5min":6.9089999999999998,"15min":12.779},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.589,"5min":1.22,"15min":1.571},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":6.0549999999999997,"5min":9.1370000000000005,"15min":14.896000000000001},"last":0.80800000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823126,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2760316,"kb_used_data":2158196,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":90984836,"statfs":{"total":95995035648,"available":93168472064,"internally_reserved":0,"allocated":2209992704,"data_stored":2209452240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.23,"15min":2.5830000000000002},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":3.9060000000000001,"5min":83.197000000000003,"15min":83.197000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.6040000000000001,"5min":2.3999999999999999,"15min":2.464},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":16.558,"5min":83.563999999999993,"15min":83.563999999999993},"last":3.5579999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:35:36 2023","interfaces":[{"interface":"back","average":{"1min":5.3719999999999999,"5min":2.1899999999999999,"15min":2.681},"min":{"1min":0.34599999999999997,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":62.494,"5min":62.494,"15min":161.71199999999999},"last":4.5970000000000004},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.6339999999999999,"15min":2.9700000000000002},"min":{"1min":0.52300000000000002,"5min":0.33600000000000002,"15min":0.30299999999999999},"max":{"1min":62.613999999999997,"5min":62.613999999999997,"15min":162.50200000000001},"last":1.0369999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:35:47 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.4410000000000001,"15min":2.2570000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":4.4859999999999998,"5min":5.2039999999999997,"15min":54.981999999999999},"last":3.7679999999999998},{"interface":"front","average":{"1min":1.8740000000000001,"5min":1.597,"15min":2.194},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.5640000000000001,"5min":9.8889999999999993,"15min":59.082999999999998},"last":0.35899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:35:23 2023","interfaces":[{"interface":"back","average":{"1min":9.5399999999999991,"5min":4.1459999999999999,"15min":3.3860000000000001},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.29199999999999998},"max":{"1min":62.316000000000003,"5min":84.174000000000007,"15min":84.174000000000007},"last":4.282},{"interface":"front","average":{"1min":8.6430000000000007,"5min":4.0129999999999999,"15min":3.1930000000000001},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":62.094000000000001,"5min":83.114999999999995,"15min":83.114999999999995},"last":0.81599999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:35:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.492,"15min":1.8080000000000001},"min":{"1min":0.36599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.438000000000001,"5min":10.438000000000001,"15min":17.667999999999999},"last":2.5640000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.198,"15min":1.4810000000000001},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.308},"max":{"1min":3.0419999999999998,"5min":5.3040000000000003,"15min":16.181999999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.4099999999999999,"15min":1.6890000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2170000000000001,"5min":8.8490000000000002,"15min":17.454999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.28,"15min":1.548},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":3.3620000000000001,"5min":6.4219999999999997,"15min":12.365},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.288,"15min":1.556},"min":{"1min":0.442,"5min":0.372,"15min":0.36899999999999999},"max":{"1min":4.3319999999999999,"5min":5.7190000000000003,"15min":12.711},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.3200000000000001,"15min":1.367},"min":{"1min":0.38,"5min":0.374,"15min":0.374},"max":{"1min":4.5170000000000003,"5min":5.5919999999999996,"15min":10.465999999999999},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1567293440,"data_stored":1567293440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1854865408,"data_stored":1854865408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1607864320,"data_stored":1607860248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1604190208,"data_stored":1604190208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1367605248,"data_stored":1367605248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2192773120,"data_stored":2192769048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1355284480,"data_stored":1355284480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1363611648,"data_stored":1363607576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:36:34.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T06:36:34.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:34.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:34.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:35.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:35.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:35.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:35.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:35.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:35.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:35.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:35.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:35.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:35.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:35.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:35.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:35.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:35.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:35.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:35.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:36.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:36.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:36.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:36.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:36 smithi155 bash[59495]: cluster 2023-10-09T06:36:35.179764+0000 mgr.y (mgr.25723) 957 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:36:36.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:36.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:36.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:36.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:36 smithi125 bash[77299]: cluster 2023-10-09T06:36:35.179764+0000 mgr.y (mgr.25723) 957 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:36:36.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:36 smithi125 bash[80002]: cluster 2023-10-09T06:36:35.179764+0000 mgr.y (mgr.25723) 957 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:36:36.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:36.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:36.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:36.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:36:36.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:36:36.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:36.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:37.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:37.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:37.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:37.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:37.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:37.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:37.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:37.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:37.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:37.599 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:37.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:37.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:37.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:37.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:37.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:37.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:37.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:38.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:38.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:38.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:38.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:38.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:38.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:38.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:38 smithi125 bash[77299]: cluster 2023-10-09T06:36:37.180326+0000 mgr.y (mgr.25723) 958 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:38.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:38 smithi125 bash[80002]: cluster 2023-10-09T06:36:37.180326+0000 mgr.y (mgr.25723) 958 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:38.496 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:38.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:38.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:38.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:38 smithi155 bash[59495]: cluster 2023-10-09T06:36:37.180326+0000 mgr.y (mgr.25723) 958 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:38.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:38.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:38.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:38.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:38.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:38.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:38.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:38.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:38.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:39.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:39.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:39.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:39.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:39.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:39.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:39.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:39.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:39.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:39.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:39.729 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:39.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:39.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:39.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:39.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:39.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:40.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:40.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:40.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:40.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:39 smithi155 bash[59495]: cluster 2023-10-09T06:36:39.180841+0000 mgr.y (mgr.25723) 959 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:40.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:40.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:40.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:40.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:40.544 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:40.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:39 smithi125 bash[77299]: cluster 2023-10-09T06:36:39.180841+0000 mgr.y (mgr.25723) 959 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:40.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:40.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:40.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:40.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:40.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:40 smithi125 bash[80002]: cluster 2023-10-09T06:36:39.180841+0000 mgr.y (mgr.25723) 959 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:40.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:40.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:40.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:41.062 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:41.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:41.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:41.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:41] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.33.4" 2023-10-09T06:36:41.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:41.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:41.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:41.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:41.520 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:41.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:41.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:41.765 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:41.766 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:41.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:41.796 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:41.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:41.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:41.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:41.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:42.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:42.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:42.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:42.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:42.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:42.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:42 smithi155 bash[59495]: cluster 2023-10-09T06:36:41.181484+0000 mgr.y (mgr.25723) 960 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:42.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:42.444 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:42.444 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:42.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:42.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:42.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:42.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:42 smithi125 bash[77299]: cluster 2023-10-09T06:36:41.181484+0000 mgr.y (mgr.25723) 960 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:42.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:42 smithi125 bash[80002]: cluster 2023-10-09T06:36:41.181484+0000 mgr.y (mgr.25723) 960 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:42.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:42.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:42.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:42.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:42.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:42.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:43.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:43.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:43.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:43.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:43.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:43.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:43.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:43.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:43.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:36:43.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:43.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:43.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:43.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:43.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:44.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:44.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:44.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:44.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:44.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:44.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:44.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:44.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:44.534 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:44 smithi125 bash[80002]: cluster 2023-10-09T06:36:43.182851+0000 mgr.y (mgr.25723) 961 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:44.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:44 smithi125 bash[77299]: cluster 2023-10-09T06:36:43.182851+0000 mgr.y (mgr.25723) 961 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:44.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:44.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:44 smithi155 bash[59495]: cluster 2023-10-09T06:36:43.182851+0000 mgr.y (mgr.25723) 961 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-09T06:36:44.663 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:44.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:44.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:44.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:44.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:44.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:45.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:45.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:45.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:45.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:45.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:45.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:45.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:45.452 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:45.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:45.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:45.581 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:45.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:45.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:45.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:45.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:45.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:45.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:45.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:45.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:46.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:46.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:46.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:46.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:46.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:46.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:46 smithi155 bash[59495]: cluster 2023-10-09T06:36:45.183531+0000 mgr.y (mgr.25723) 962 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:36:46.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:46.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:46.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:46 smithi125 bash[77299]: cluster 2023-10-09T06:36:45.183531+0000 mgr.y (mgr.25723) 962 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:36:46.591 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:46 smithi125 bash[80002]: cluster 2023-10-09T06:36:45.183531+0000 mgr.y (mgr.25723) 962 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:36:46.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:46.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:46.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:46.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:46.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:46.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:46.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:46.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:46.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:47.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:47.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:47.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:47.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.378 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:47.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:47.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:47.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:47.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:47.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:47.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:47.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:47.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:48.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:48.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:48.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:48.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:48.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:48 smithi125 bash[77299]: cluster 2023-10-09T06:36:47.184082+0000 mgr.y (mgr.25723) 963 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:36:48.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:48 smithi125 bash[80002]: cluster 2023-10-09T06:36:47.184082+0000 mgr.y (mgr.25723) 963 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:36:48.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:48.430 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:48.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:48.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:48.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:48 smithi155 bash[59495]: cluster 2023-10-09T06:36:47.184082+0000 mgr.y (mgr.25723) 963 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-09T06:36:48.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:48.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:48.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:48.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:48.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:48.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:49.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:49.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:49.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:49.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:49.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:49.493 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:49 smithi155 bash[59495]: audit 2023-10-09T06:36:48.592183+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:49.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:49.538 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:49.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:49 smithi125 bash[77299]: audit 2023-10-09T06:36:48.592183+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:49.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:49 smithi125 bash[80002]: audit 2023-10-09T06:36:48.592183+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:49.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:49.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:49.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:49.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:49.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:50.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:50.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:50.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:50.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:50.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:50.311 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:50 smithi155 bash[59495]: cluster 2023-10-09T06:36:49.184629+0000 mgr.y (mgr.25723) 964 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:36:50.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:50.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:50.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:50.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:50.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:50 smithi125 bash[77299]: cluster 2023-10-09T06:36:49.184629+0000 mgr.y (mgr.25723) 964 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:36:50.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:50 smithi125 bash[80002]: cluster 2023-10-09T06:36:49.184629+0000 mgr.y (mgr.25723) 964 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-10-09T06:36:50.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:50.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:50.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:50.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:50.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:50.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:51.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:51.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:51.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:51.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:51.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.296 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:51 smithi155 bash[59495]: audit 2023-10-09T06:36:50.628686+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:36:51.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:51.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:51.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:51 smithi125 bash[77299]: audit 2023-10-09T06:36:50.628686+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:36:51.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:51 smithi125 bash[80002]: audit 2023-10-09T06:36:50.628686+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:36:51.561 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:36:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:36:51] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:36:51.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:51.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:51.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:51.849 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:51.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:51.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:51.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:52.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:52.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:52.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:52.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:52.297 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:52 smithi155 bash[59495]: cluster 2023-10-09T06:36:51.185137+0000 mgr.y (mgr.25723) 965 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:52.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:36:52.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:52.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:52.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:52.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:52 smithi125 bash[77299]: cluster 2023-10-09T06:36:51.185137+0000 mgr.y (mgr.25723) 965 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:52.581 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:52 smithi125 bash[80002]: cluster 2023-10-09T06:36:51.185137+0000 mgr.y (mgr.25723) 965 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:36:52.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:52.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:52.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:52.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:52.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:52.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:52.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:52.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:52.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:52.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:53.115 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:53.115 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:53.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:53.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:53.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:53.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:53.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:53 smithi125 bash[77299]: audit 2023-10-09T06:36:52.353884+0000 mon.a (mon.0) 1160 : audit 1 from='client.? 172.21.15.125:0/3965849516' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:36:53.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:53 smithi125 bash[80002]: audit 2023-10-09T06:36:52.353884+0000 mon.a (mon.0) 1160 : audit 1 from='client.? 172.21.15.125:0/3965849516' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:36:53.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:53.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:53 smithi155 bash[59495]: audit 2023-10-09T06:36:52.353884+0000 mon.a (mon.0) 1160 : audit 1 from='client.? 172.21.15.125:0/3965849516' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:36:53.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:53.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:53.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:53.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:53.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:53.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:53.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:53.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:54.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:54.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:54.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:54.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:36:54.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:54.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:54.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:54.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:54.384 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:54 smithi125 bash[77299]: cluster 2023-10-09T06:36:53.186943+0000 mgr.y (mgr.25723) 966 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:36:54.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:54 smithi125 bash[80002]: cluster 2023-10-09T06:36:53.186943+0000 mgr.y (mgr.25723) 966 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:36:54.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:54.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:54.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:54.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:54.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:54 smithi155 bash[59495]: cluster 2023-10-09T06:36:53.186943+0000 mgr.y (mgr.25723) 966 : cluster 0 pgmap v980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T06:36:54.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:54.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:54.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:54.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:54.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:54.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:54.944 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:54.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:54.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:55.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:55.069 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:36:55.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:55.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:55.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:55.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:55.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:55.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:55.562 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:36:55.606 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:36:55.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:55.653 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:36:55.695 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:36:55.741 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:36:55.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:55.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:55.792 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:55.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:55.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:56.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:56.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:56.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:56.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:56.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:56.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:56.335 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:36:56.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[77299]: cluster 2023-10-09T06:36:55.187612+0000 mgr.y (mgr.25723) 967 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:36:56.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[77299]: audit 2023-10-09T06:36:55.516782+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.125:0/2392880889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:56.375 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[77299]: audit 2023-10-09T06:36:55.925085+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:56.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[80002]: cluster 2023-10-09T06:36:55.187612+0000 mgr.y (mgr.25723) 967 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:36:56.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[80002]: audit 2023-10-09T06:36:55.516782+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.125:0/2392880889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:56.375 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:56 smithi125 bash[80002]: audit 2023-10-09T06:36:55.925085+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:56.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:56.447 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:56.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:56.596 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:56 smithi155 bash[59495]: cluster 2023-10-09T06:36:55.187612+0000 mgr.y (mgr.25723) 967 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T06:36:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:56 smithi155 bash[59495]: audit 2023-10-09T06:36:55.516782+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.125:0/2392880889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:56 smithi155 bash[59495]: audit 2023-10-09T06:36:55.925085+0000 mon.a (mon.0) 1161 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:56.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:56.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:56.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:56.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:56.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:56.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:56.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:57.021 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:57.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:57.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:57.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:57.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.304052+0000 mon.a (mon.0) 1162 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: audit 2023-10-09T06:36:56.317006+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.317123+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.326014+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.327201+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.330082+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.334627+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.334658+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:57 smithi155 bash[59495]: cluster 2023-10-09T06:36:56.336694+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:57.472 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:57.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:57.586 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.304052+0000 mon.a (mon.0) 1162 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: audit 2023-10-09T06:36:56.317006+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.317123+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.326014+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.327201+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.330082+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.334627+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.334658+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[80002]: cluster 2023-10-09T06:36:56.336694+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.304052+0000 mon.a (mon.0) 1162 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:57.587 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: audit 2023-10-09T06:36:56.317006+0000 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.125:0/831909429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.317123+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.326014+0000 osd.4 (osd.4) 130 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.327201+0000 osd.4 (osd.4) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.330082+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.334627+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.334658+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:57 smithi125 bash[77299]: cluster 2023-10-09T06:36:56.336694+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e205 with expected crc 2023-10-09T06:36:57.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:57.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:57.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:57.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:57.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:57.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:58.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:58.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:58.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:58.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:58.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[80002]: cluster 2023-10-09T06:36:57.188057+0000 mgr.y (mgr.25723) 968 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:36:58.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[80002]: audit 2023-10-09T06:36:58.293760+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[80002]: audit 2023-10-09T06:36:58.304018+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:58.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:58.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:58.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:58 smithi155 bash[59495]: cluster 2023-10-09T06:36:57.188057+0000 mgr.y (mgr.25723) 968 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:36:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:58 smithi155 bash[59495]: audit 2023-10-09T06:36:58.293760+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:58 smithi155 bash[59495]: audit 2023-10-09T06:36:58.304018+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:58.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[77299]: cluster 2023-10-09T06:36:57.188057+0000 mgr.y (mgr.25723) 968 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:36:58.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[77299]: audit 2023-10-09T06:36:58.293760+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:58 smithi125 bash[77299]: audit 2023-10-09T06:36:58.304018+0000 mon.a (mon.0) 1166 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:36:58.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:58.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:58.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:58.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:58.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:59.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:59.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:59.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:59.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:59.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:59.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:59.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:59.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:59.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:59.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:59.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:36:59.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:59.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:00.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:37:00.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:00.071 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:37:00.071 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:37:00.071 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-10-09T06:37:00.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:37:00.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:36:59 smithi155 bash[59495]: cluster 2023-10-09T06:36:59.190935+0000 mgr.y (mgr.25723) 969 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:37:00.169 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:00.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:36:59 smithi125 bash[77299]: cluster 2023-10-09T06:36:59.190935+0000 mgr.y (mgr.25723) 969 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:37:00.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:36:59 smithi125 bash[80002]: cluster 2023-10-09T06:36:59.190935+0000 mgr.y (mgr.25723) 969 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T06:37:00.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:00.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:00.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:00.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:00.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:00.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:37:00.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:00.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:00.824 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:00.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:00.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:00.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:01.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:01.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:01.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:01.201 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:01.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:01.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:01.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:01.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:01.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:01.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:01.553 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:01] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:37:01.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:01.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:01.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:01.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:01.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:01.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:01.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:02.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:02.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:02.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:02.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:02.153 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:02.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:02.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:02.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:02.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:02.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:02.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:02.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:02.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:02.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:02 smithi125 bash[77299]: cluster 2023-10-09T06:37:01.191610+0000 mgr.y (mgr.25723) 970 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:37:02.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:02 smithi125 bash[80002]: cluster 2023-10-09T06:37:01.191610+0000 mgr.y (mgr.25723) 970 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:37:02.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:02 smithi155 bash[59495]: cluster 2023-10-09T06:37:01.191610+0000 mgr.y (mgr.25723) 970 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T06:37:02.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:02.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:02.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:03.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:03.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:03.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:03.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:03.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:03.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:03.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:03.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:03.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:03.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:03.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:03.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:03.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:03.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:03.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:03.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:04.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:04.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:04.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:04.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:04.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:04.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:04 smithi155 bash[59495]: cluster 2023-10-09T06:37:03.192144+0000 mgr.y (mgr.25723) 971 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:37:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:04 smithi155 bash[59495]: audit 2023-10-09T06:37:03.596698+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:04 smithi155 bash[59495]: audit 2023-10-09T06:37:03.599322+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:04 smithi155 bash[59495]: audit 2023-10-09T06:37:03.969677+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:04 smithi155 bash[59495]: audit 2023-10-09T06:37:03.982465+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:04.443 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[77299]: cluster 2023-10-09T06:37:03.192144+0000 mgr.y (mgr.25723) 971 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[77299]: audit 2023-10-09T06:37:03.596698+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[77299]: audit 2023-10-09T06:37:03.599322+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[77299]: audit 2023-10-09T06:37:03.969677+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[77299]: audit 2023-10-09T06:37:03.982465+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[80002]: cluster 2023-10-09T06:37:03.192144+0000 mgr.y (mgr.25723) 971 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[80002]: audit 2023-10-09T06:37:03.596698+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[80002]: audit 2023-10-09T06:37:03.599322+0000 mon.a (mon.0) 1168 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:04.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[80002]: audit 2023-10-09T06:37:03.969677+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:04 smithi125 bash[80002]: audit 2023-10-09T06:37:03.982465+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:04.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:04.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:04.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:04.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:04.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:04.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:04.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:04.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:04.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:04.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:05.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:05.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:05.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:05.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:05.504 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:05.504 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:05.511 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:05.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[77299]: audit 2023-10-09T06:37:04.761571+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[77299]: audit 2023-10-09T06:37:04.762652+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[77299]: audit 2023-10-09T06:37:04.771200+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[80002]: audit 2023-10-09T06:37:04.761571+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[80002]: audit 2023-10-09T06:37:04.762652+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:05.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:05 smithi125 bash[80002]: audit 2023-10-09T06:37:04.771200+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:05.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:05.634 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:05.642 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:05 smithi155 bash[59495]: audit 2023-10-09T06:37:04.761571+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:05 smithi155 bash[59495]: audit 2023-10-09T06:37:04.762652+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:05 smithi155 bash[59495]: audit 2023-10-09T06:37:04.771200+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:05.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:05.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:05.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:05.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:05.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:05.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:06.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:06.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:06.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:06.222 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:06.269 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:06 smithi155 bash[59495]: cluster 2023-10-09T06:37:05.192976+0000 mgr.y (mgr.25723) 972 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:06.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:06.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:06.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:06.475 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:06 smithi125 bash[77299]: cluster 2023-10-09T06:37:05.192976+0000 mgr.y (mgr.25723) 972 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:06.476 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:06 smithi125 bash[80002]: cluster 2023-10-09T06:37:05.192976+0000 mgr.y (mgr.25723) 972 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:06.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:06.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:06.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:06.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:06.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:06.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:06.893 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:06.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:06.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:07.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:07.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:07.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:07.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:07.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:07.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:07.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:07.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:07.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:07.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:07.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:07.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:07.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:07.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:07.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:07.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:08.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:08.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:08.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:08.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:08.275 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:08 smithi125 bash[77299]: cluster 2023-10-09T06:37:07.193557+0000 mgr.y (mgr.25723) 973 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:37:08.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:08.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:08.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:08.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:08.616 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:08 smithi125 bash[80002]: cluster 2023-10-09T06:37:07.193557+0000 mgr.y (mgr.25723) 973 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:37:08.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:08.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:08.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:08 smithi155 bash[59495]: cluster 2023-10-09T06:37:07.193557+0000 mgr.y (mgr.25723) 973 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:37:08.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:08.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:08.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:08.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:08.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:08.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:09.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:09.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:09.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:09.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:09.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:09.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:09.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:09.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:09.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:09.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:09.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:09.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:09.783 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:37:09.783 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:37:09.783 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T06:37:09.783 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:37:09.877 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:09.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:09.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:10.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.030 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:09 smithi155 bash[59495]: cluster 2023-10-09T06:37:09.194305+0000 mgr.y (mgr.25723) 974 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T06:37:10.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:10.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:09 smithi125 bash[80002]: cluster 2023-10-09T06:37:09.194305+0000 mgr.y (mgr.25723) 974 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T06:37:10.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:09 smithi125 bash[77299]: cluster 2023-10-09T06:37:09.194305+0000 mgr.y (mgr.25723) 974 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-09T06:37:10.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:10.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:10.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:10.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:10.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:10.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:10.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:10.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:10.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:10.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:10.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:10.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:10.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:10.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:10.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:11.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:11.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:11.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:11.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:11.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:11.391 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:11] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-10-09T06:37:11.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:11.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:11.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:11.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:11.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:11.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:11.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:11.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:11.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:11.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:11.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:12.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:12.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:12.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:12.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:12.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:12.364 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:12.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:12.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:12 smithi125 bash[77299]: cluster 2023-10-09T06:37:11.194943+0000 mgr.y (mgr.25723) 975 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:12.505 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:12 smithi125 bash[80002]: cluster 2023-10-09T06:37:11.194943+0000 mgr.y (mgr.25723) 975 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:12.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:12.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:12.655 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:12 smithi155 bash[59495]: cluster 2023-10-09T06:37:11.194943+0000 mgr.y (mgr.25723) 975 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:12.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:12.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:12.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:12.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:12.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:12.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:12.960 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:12.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:13.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:13.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:13.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:13.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:13.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:13.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:13.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:13.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:37:13.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:13.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:13.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:13.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:13.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:13.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:13.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:14.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:14.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:14.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:14.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:14.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:14.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:14.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:14.521 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:14.530 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:14 smithi125 bash[77299]: cluster 2023-10-09T06:37:13.195374+0000 mgr.y (mgr.25723) 976 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:37:14.530 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:14 smithi125 bash[80002]: cluster 2023-10-09T06:37:13.195374+0000 mgr.y (mgr.25723) 976 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:37:14.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:14.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:14 smithi155 bash[59495]: cluster 2023-10-09T06:37:13.195374+0000 mgr.y (mgr.25723) 976 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-09T06:37:14.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:14.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:14.804 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:14.828 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:14.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:14.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:14.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:14.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:15.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:15.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.224 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:15.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:15.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:15.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:15.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:15.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:15.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:15.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:15.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:15.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:15.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:15.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.893 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:15.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:15.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:16.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:16.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:16.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:16.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:16.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:16.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:16.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:16.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:16 smithi155 bash[59495]: cluster 2023-10-09T06:37:15.196029+0000 mgr.y (mgr.25723) 977 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:37:16.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:16.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:16.541 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:16 smithi125 bash[77299]: cluster 2023-10-09T06:37:15.196029+0000 mgr.y (mgr.25723) 977 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:37:16.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:16 smithi125 bash[80002]: cluster 2023-10-09T06:37:15.196029+0000 mgr.y (mgr.25723) 977 : cluster 0 pgmap v992: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-09T06:37:16.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:16.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:16.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:16.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:16.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:16.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:16.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:16.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:17.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:17.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:17.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:17.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:17.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:17.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:17.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:17.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:17.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:17.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:17.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:17.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:17.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:18.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:18.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:18.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:18.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:18 smithi125 bash[77299]: cluster 2023-10-09T06:37:17.196566+0000 mgr.y (mgr.25723) 978 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:18.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:18 smithi125 bash[80002]: cluster 2023-10-09T06:37:17.196566+0000 mgr.y (mgr.25723) 978 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:18.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:18.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:18.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:18.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:18.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:18.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:18 smithi155 bash[59495]: cluster 2023-10-09T06:37:17.196566+0000 mgr.y (mgr.25723) 978 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:37:18.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:18.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:18.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.883 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:18.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:18.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:19.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:19.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:19.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:19.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:19.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:19.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:19.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:19.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:19.523 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:19 smithi125 bash[77299]: audit 2023-10-09T06:37:18.592858+0000 mon.a (mon.0) 1174 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:19.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:19 smithi125 bash[80002]: audit 2023-10-09T06:37:18.592858+0000 mon.a (mon.0) 1174 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:19.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:19 smithi155 bash[59495]: audit 2023-10-09T06:37:18.592858+0000 mon.a (mon.0) 1174 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:19.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:19.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:19.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:19.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:19.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:19.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:19.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:19.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:20.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:20.087 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:20.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.220 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:20.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:20.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:20.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:20 smithi125 bash[77299]: cluster 2023-10-09T06:37:19.198320+0000 mgr.y (mgr.25723) 979 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T06:37:20.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:20 smithi125 bash[80002]: cluster 2023-10-09T06:37:19.198320+0000 mgr.y (mgr.25723) 979 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T06:37:20.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:20.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:20.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:20 smithi155 bash[59495]: cluster 2023-10-09T06:37:19.198320+0000 mgr.y (mgr.25723) 979 : cluster 0 pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T06:37:20.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:20.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:20.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:20.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:20.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:20.908 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:21.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:21.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:21.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:21.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:21.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.424 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:21] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:37:21.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:21.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:21.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:21.666 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:21.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:21.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.958 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:21.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:21.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:22.061 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:22.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:22.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:22.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:22.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:22.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:22.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:22 smithi125 bash[77299]: cluster 2023-10-09T06:37:21.198982+0000 mgr.y (mgr.25723) 980 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:22.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:22 smithi125 bash[80002]: cluster 2023-10-09T06:37:21.198982+0000 mgr.y (mgr.25723) 980 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:22.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:22.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:22 smithi155 bash[59495]: cluster 2023-10-09T06:37:21.198982+0000 mgr.y (mgr.25723) 980 : cluster 0 pgmap v995: 89 pgs: 89 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:22.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:22.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:22.569 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:37:22.575 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:37:22.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:22.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:22.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:22.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:22.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:22.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:22.911 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:22.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:23.012 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:23.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:23.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:23.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:23.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:23.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:23.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:23.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:23.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:23.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:24.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:24.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:24.379 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:24.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:24.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:24.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:24.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:24 smithi125 bash[77299]: audit 2023-10-09T06:37:22.541273+0000 mgr.y (mgr.25723) 981 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:24.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:24 smithi125 bash[77299]: cluster 2023-10-09T06:37:23.199495+0000 mgr.y (mgr.25723) 982 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:37:24.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:24 smithi125 bash[80002]: audit 2023-10-09T06:37:22.541273+0000 mgr.y (mgr.25723) 981 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:24.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:24 smithi125 bash[80002]: cluster 2023-10-09T06:37:23.199495+0000 mgr.y (mgr.25723) 982 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:37:24.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:24 smithi155 bash[59495]: audit 2023-10-09T06:37:22.541273+0000 mgr.y (mgr.25723) 981 : audit 0 from='client.45491 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:24 smithi155 bash[59495]: cluster 2023-10-09T06:37:23.199495+0000 mgr.y (mgr.25723) 982 : cluster 0 pgmap v996: 89 pgs: 89 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:37:24.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:24.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:24.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:24.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:24.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:24.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:25.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:25.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:25.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:25.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:25.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:25.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:25.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:25.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:25.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.602 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":995,"stamp":"2023-10-09T06:37:21.198513+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2018056139,"num_objects":30749,"num_object_clones":0,"num_object_copies":92247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30749,"num_whiteouts":0,"num_read":6290,"num_read_kb":13749,"num_write":188517,"num_write_kb":6995320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":2380802,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149081,"ondisk_log_size":149081,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9534860,"kb_used_data":6017036,"kb_used_omap":30,"kb_used_meta":3517537,"kb_avail":740426356,"statfs":{"total":767960285184,"available":758196588544,"internally_reserved":0,"allocated":6161444864,"data_stored":6155867973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":3601957999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,7,9,5,3,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-530382848,"num_objects":-8093,"num_object_clones":0,"num_object_copies":-24279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8093,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004674"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993031+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:36:31.993031+0000","last_peered":"2023-10-09T06:36:31.993031+0000","last_clean":"2023-10-09T06:36:31.993031+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:36:31.993031+0000","last_undegraded":"2023-10-09T06:36:31.993031+0000","last_fullsized":"2023-10-09T06:36:31.993031+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":182,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.329958+0000","last_change":"2023-10-09T06:14:42.681459+0000","last_active":"2023-10-09T06:36:56.329958+0000","last_peered":"2023-10-09T06:36:56.329958+0000","last_clean":"2023-10-09T06:36:56.329958+0000","last_became_active":"2023-10-09T06:14:42.680375+0000","last_became_peered":"2023-10-09T06:14:42.680375+0000","last_unstale":"2023-10-09T06:36:56.329958+0000","last_undegraded":"2023-10-09T06:36:56.329958+0000","last_fullsized":"2023-10-09T06:36:56.329958+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:46.525898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":53,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.982477+0000","last_change":"2023-10-09T06:35:45.655708+0000","last_active":"2023-10-09T06:36:31.982477+0000","last_peered":"2023-10-09T06:36:31.982477+0000","last_clean":"2023-10-09T06:36:31.982477+0000","last_became_active":"2023-10-09T06:35:45.655323+0000","last_became_peered":"2023-10-09T06:35:45.655323+0000","last_unstale":"2023-10-09T06:36:31.982477+0000","last_undegraded":"2023-10-09T06:36:31.982477+0000","last_fullsized":"2023-10-09T06:36:31.982477+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:00.683127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.322226+0000","last_change":"2023-10-09T06:18:59.799836+0000","last_active":"2023-10-09T06:36:56.322226+0000","last_peered":"2023-10-09T06:36:56.322226+0000","last_clean":"2023-10-09T06:36:56.322226+0000","last_became_active":"2023-10-09T06:18:59.799249+0000","last_became_peered":"2023-10-09T06:18:59.799249+0000","last_unstale":"2023-10-09T06:36:56.322226+0000","last_undegraded":"2023-10-09T06:36:56.322226+0000","last_fullsized":"2023-10-09T06:36:56.322226+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:09.925240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":198,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983161+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:36:31.983161+0000","last_peered":"2023-10-09T06:36:31.983161+0000","last_clean":"2023-10-09T06:36:31.983161+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:36:31.983161+0000","last_undegraded":"2023-10-09T06:36:31.983161+0000","last_fullsized":"2023-10-09T06:36:31.983161+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984496+0000","last_change":"2023-10-09T06:28:31.033925+0000","last_active":"2023-10-09T06:36:31.984496+0000","last_peered":"2023-10-09T06:36:31.984496+0000","last_clean":"2023-10-09T06:36:31.984496+0000","last_became_active":"2023-10-09T06:28:31.033544+0000","last_became_peered":"2023-10-09T06:28:31.033544+0000","last_unstale":"2023-10-09T06:36:31.984496+0000","last_undegraded":"2023-10-09T06:36:31.984496+0000","last_fullsized":"2023-10-09T06:36:31.984496+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984526+0000","last_change":"2023-10-09T06:14:42.683925+0000","last_active":"2023-10-09T06:36:31.984526+0000","last_peered":"2023-10-09T06:36:31.984526+0000","last_clean":"2023-10-09T06:36:31.984526+0000","last_became_active":"2023-10-09T06:14:42.683600+0000","last_became_peered":"2023-10-09T06:14:42.683600+0000","last_unstale":"2023-10-09T06:36:31.984526+0000","last_undegraded":"2023-10-09T06:36:31.984526+0000","last_fullsized":"2023-10-09T06:36:31.984526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:32.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983241+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:36:31.983241+0000","last_peered":"2023-10-09T06:36:31.983241+0000","last_clean":"2023-10-09T06:36:31.983241+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:36:31.983241+0000","last_undegraded":"2023-10-09T06:36:31.983241+0000","last_fullsized":"2023-10-09T06:36:31.983241+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994112+0000","last_change":"2023-10-09T06:18:59.802209+0000","last_active":"2023-10-09T06:36:31.994112+0000","last_peered":"2023-10-09T06:36:31.994112+0000","last_clean":"2023-10-09T06:36:31.994112+0000","last_became_active":"2023-10-09T06:18:59.801848+0000","last_became_peered":"2023-10-09T06:18:59.801848+0000","last_unstale":"2023-10-09T06:36:31.994112+0000","last_undegraded":"2023-10-09T06:36:31.994112+0000","last_fullsized":"2023-10-09T06:36:31.994112+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.982400+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:36:31.982400+0000","last_peered":"2023-10-09T06:36:31.982400+0000","last_clean":"2023-10-09T06:36:31.982400+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:36:31.982400+0000","last_undegraded":"2023-10-09T06:36:31.982400+0000","last_fullsized":"2023-10-09T06:36:31.982400+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":103,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.003415+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:36:32.003415+0000","last_peered":"2023-10-09T06:36:32.003415+0000","last_clean":"2023-10-09T06:36:32.003415+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:36:32.003415+0000","last_undegraded":"2023-10-09T06:36:32.003415+0000","last_fullsized":"2023-10-09T06:36:32.003415+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983606+0000","last_change":"2023-10-09T06:35:45.663547+0000","last_active":"2023-10-09T06:36:31.983606+0000","last_peered":"2023-10-09T06:36:31.983606+0000","last_clean":"2023-10-09T06:36:31.983606+0000","last_became_active":"2023-10-09T06:35:45.663179+0000","last_became_peered":"2023-10-09T06:35:45.663179+0000","last_unstale":"2023-10-09T06:36:31.983606+0000","last_undegraded":"2023-10-09T06:36:31.983606+0000","last_fullsized":"2023-10-09T06:36:31.983606+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":253,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993220+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:36:31.993220+0000","last_peered":"2023-10-09T06:36:31.993220+0000","last_clean":"2023-10-09T06:36:31.993220+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:36:31.993220+0000","last_undegraded":"2023-10-09T06:36:31.993220+0000","last_fullsized":"2023-10-09T06:36:31.993220+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993462+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:36:31.993462+0000","last_peered":"2023-10-09T06:36:31.993462+0000","last_clean":"2023-10-09T06:36:31.993462+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:36:31.993462+0000","last_undegraded":"2023-10-09T06:36:31.993462+0000","last_fullsized":"2023-10-09T06:36:31.993462+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":103,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.002529+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:36:32.002529+0000","last_peered":"2023-10-09T06:36:32.002529+0000","last_clean":"2023-10-09T06:36:32.002529+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:36:32.002529+0000","last_undegraded":"2023-10-09T06:36:32.002529+0000","last_fullsized":"2023-10-09T06:36:32.002529+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":200,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983085+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:36:31.983085+0000","last_peered":"2023-10-09T06:36:31.983085+0000","last_clean":"2023-10-09T06:36:31.983085+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:36:31.983085+0000","last_undegraded":"2023-10-09T06:36:31.983085+0000","last_fullsized":"2023-10-09T06:36:31.983085+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.011853+0000","last_change":"2023-10-09T06:18:59.793272+0000","last_active":"2023-10-09T06:36:34.011853+0000","last_peered":"2023-10-09T06:36:34.011853+0000","last_clean":"2023-10-09T06:36:34.011853+0000","last_became_active":"2023-10-09T06:18:59.792922+0000","last_became_peered":"2023-10-09T06:18:59.792922+0000","last_unstale":"2023-10-09T06:36:34.011853+0000","last_undegraded":"2023-10-09T06:36:34.011853+0000","last_fullsized":"2023-10-09T06:36:34.011853+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:18.554255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994114+0000","last_change":"2023-10-09T06:18:59.807634+0000","last_active":"2023-10-09T06:36:31.994114+0000","last_peered":"2023-10-09T06:36:31.994114+0000","last_clean":"2023-10-09T06:36:31.994114+0000","last_became_active":"2023-10-09T06:18:59.807230+0000","last_became_peered":"2023-10-09T06:18:59.807230+0000","last_unstale":"2023-10-09T06:36:31.994114+0000","last_undegraded":"2023-10-09T06:36:31.994114+0000","last_fullsized":"2023-10-09T06:36:31.994114+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":115,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.002636+0000","last_change":"2023-10-09T06:28:31.027079+0000","last_active":"2023-10-09T06:36:32.002636+0000","last_peered":"2023-10-09T06:36:32.002636+0000","last_clean":"2023-10-09T06:36:32.002636+0000","last_became_active":"2023-10-09T06:28:31.026691+0000","last_became_peered":"2023-10-09T06:28:31.026691+0000","last_unstale":"2023-10-09T06:36:32.002636+0000","last_undegraded":"2023-10-09T06:36:32.002636+0000","last_fullsized":"2023-10-09T06:36:32.002636+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":179,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.985554+0000","last_change":"2023-10-09T06:18:59.799603+0000","last_active":"2023-10-09T06:36:31.985554+0000","last_peered":"2023-10-09T06:36:31.985554+0000","last_clean":"2023-10-09T06:36:31.985554+0000","last_became_active":"2023-10-09T06:18:59.798805+0000","last_became_peered":"2023-10-09T06:18:59.798805+0000","last_unstale":"2023-10-09T06:36:31.985554+0000","last_undegraded":"2023-10-09T06:36:31.985554+0000","last_fullsized":"2023-10-09T06:36:31.985554+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:05:13.599029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":55,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983942+0000","last_change":"2023-10-09T06:28:31.473895+0000","last_active":"2023-10-09T06:36:31.983942+0000","last_peered":"2023-10-09T06:36:31.983942+0000","last_clean":"2023-10-09T06:36:31.983942+0000","last_became_active":"2023-10-09T06:28:31.033788+0000","last_became_peered":"2023-10-09T06:28:31.033788+0000","last_unstale":"2023-10-09T06:36:31.983942+0000","last_undegraded":"2023-10-09T06:36:31.983942+0000","last_fullsized":"2023-10-09T06:36:31.983942+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.989523+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:36:31.989523+0000","last_peered":"2023-10-09T06:36:31.989523+0000","last_clean":"2023-10-09T06:36:31.989523+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:36:31.989523+0000","last_undegraded":"2023-10-09T06:36:31.989523+0000","last_fullsized":"2023-10-09T06:36:31.989523+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"205'11775","reported_seq":11796,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.895685+0000","last_change":"2023-10-09T06:33:33.324346+0000","last_active":"2023-10-09T06:37:19.895685+0000","last_peered":"2023-10-09T06:37:19.895685+0000","last_clean":"2023-10-09T06:37:19.895685+0000","last_became_active":"2023-10-09T06:33:33.323046+0000","last_became_peered":"2023-10-09T06:33:33.323046+0000","last_unstale":"2023-10-09T06:37:19.895685+0000","last_undegraded":"2023-10-09T06:37:19.895685+0000","last_fullsized":"2023-10-09T06:37:19.895685+0000","mapping_epoch":192,"log_start":"197'1700","ondisk_log_start":"197'1700","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10075,"log_dups_size":0,"ondisk_log_size":10075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11775,"num_write_kb":436864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":101,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.002576+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:36:32.002576+0000","last_peered":"2023-10-09T06:36:32.002576+0000","last_clean":"2023-10-09T06:36:32.002576+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:36:32.002576+0000","last_undegraded":"2023-10-09T06:36:32.002576+0000","last_fullsized":"2023-10-09T06:36:32.002576+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":190,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.991242+0000","last_change":"2023-10-09T06:28:31.029529+0000","last_active":"2023-10-09T06:36:31.991242+0000","last_peered":"2023-10-09T06:36:31.991242+0000","last_clean":"2023-10-09T06:36:31.991242+0000","last_became_active":"2023-10-09T06:28:31.029118+0000","last_became_peered":"2023-10-09T06:28:31.029118+0000","last_unstale":"2023-10-09T06:36:31.991242+0000","last_undegraded":"2023-10-09T06:36:31.991242+0000","last_fullsized":"2023-10-09T06:36:31.991242+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"205'11629","reported_seq":11650,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:20.086092+0000","last_change":"2023-10-09T06:33:33.593835+0000","last_active":"2023-10-09T06:37:20.086092+0000","last_peered":"2023-10-09T06:37:20.086092+0000","last_clean":"2023-10-09T06:37:20.086092+0000","last_became_active":"2023-10-09T06:33:33.593363+0000","last_became_peered":"2023-10-09T06:33:33.593363+0000","last_unstale":"2023-10-09T06:37:20.086092+0000","last_undegraded":"2023-10-09T06:37:20.086092+0000","last_fullsized":"2023-10-09T06:37:20.086092+0000","mapping_epoch":192,"log_start":"197'2200","ondisk_log_start":"197'2200","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9429,"log_dups_size":0,"ondisk_log_size":9429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125501440,"num_objects":1915,"num_object_clones":0,"num_object_copies":5745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11629,"num_write_kb":433408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":101,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.006808+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:36:32.006808+0000","last_peered":"2023-10-09T06:36:32.006808+0000","last_clean":"2023-10-09T06:36:32.006808+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:36:32.006808+0000","last_undegraded":"2023-10-09T06:36:32.006808+0000","last_fullsized":"2023-10-09T06:36:32.006808+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992981+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:36:31.992981+0000","last_peered":"2023-10-09T06:36:31.992981+0000","last_clean":"2023-10-09T06:36:31.992981+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:36:31.992981+0000","last_undegraded":"2023-10-09T06:36:31.992981+0000","last_fullsized":"2023-10-09T06:36:31.992981+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"205'11754","reported_seq":11775,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.202341+0000","last_change":"2023-10-09T06:33:33.587458+0000","last_active":"2023-10-09T06:37:19.202341+0000","last_peered":"2023-10-09T06:37:19.202341+0000","last_clean":"2023-10-09T06:37:19.202341+0000","last_became_active":"2023-10-09T06:33:33.586581+0000","last_became_peered":"2023-10-09T06:33:33.586581+0000","last_unstale":"2023-10-09T06:37:19.202341+0000","last_undegraded":"2023-10-09T06:37:19.202341+0000","last_fullsized":"2023-10-09T06:37:19.202341+0000","mapping_epoch":192,"log_start":"197'2300","ondisk_log_start":"197'2300","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9454,"log_dups_size":0,"ondisk_log_size":9454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:40:15.486216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125173760,"num_objects":1910,"num_object_clones":0,"num_object_copies":5730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11754,"num_write_kb":437248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":127,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.003174+0000","last_change":"2023-10-09T06:28:31.127483+0000","last_active":"2023-10-09T06:36:32.003174+0000","last_peered":"2023-10-09T06:36:32.003174+0000","last_clean":"2023-10-09T06:36:32.003174+0000","last_became_active":"2023-10-09T06:28:31.026951+0000","last_became_peered":"2023-10-09T06:28:31.026951+0000","last_unstale":"2023-10-09T06:36:32.003174+0000","last_undegraded":"2023-10-09T06:36:32.003174+0000","last_fullsized":"2023-10-09T06:36:32.003174+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":45,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983523+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:36:31.983523+0000","last_peered":"2023-10-09T06:36:31.983523+0000","last_clean":"2023-10-09T06:36:31.983523+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:36:31.983523+0000","last_undegraded":"2023-10-09T06:36:31.983523+0000","last_fullsized":"2023-10-09T06:36:31.983523+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"205'11608","reported_seq":11638,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.122724+0000","last_change":"2023-10-09T06:35:45.678703+0000","last_active":"2023-10-09T06:37:17.122724+0000","last_peered":"2023-10-09T06:37:17.122724+0000","last_clean":"2023-10-09T06:37:17.122724+0000","last_became_active":"2023-10-09T06:35:45.655393+0000","last_became_peered":"2023-10-09T06:35:45.655393+0000","last_unstale":"2023-10-09T06:37:17.122724+0000","last_undegraded":"2023-10-09T06:37:17.122724+0000","last_fullsized":"2023-10-09T06:37:17.122724+0000","mapping_epoch":201,"log_start":"197'3400","ondisk_log_start":"197'3400","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:26.263052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126222336,"num_objects":1926,"num_object_clones":0,"num_object_copies":5778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11608,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994043+0000","last_change":"2023-10-09T06:28:31.030617+0000","last_active":"2023-10-09T06:36:31.994043+0000","last_peered":"2023-10-09T06:36:31.994043+0000","last_clean":"2023-10-09T06:36:31.994043+0000","last_became_active":"2023-10-09T06:28:31.030230+0000","last_became_peered":"2023-10-09T06:28:31.030230+0000","last_unstale":"2023-10-09T06:36:31.994043+0000","last_undegraded":"2023-10-09T06:36:31.994043+0000","last_fullsized":"2023-10-09T06:36:31.994043+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984031+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:36:31.984031+0000","last_peered":"2023-10-09T06:36:31.984031+0000","last_clean":"2023-10-09T06:36:31.984031+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:36:31.984031+0000","last_undegraded":"2023-10-09T06:36:31.984031+0000","last_fullsized":"2023-10-09T06:36:31.984031+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"205'11810","reported_seq":11833,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.942869+0000","last_change":"2023-10-09T06:33:33.320528+0000","last_active":"2023-10-09T06:37:16.942869+0000","last_peered":"2023-10-09T06:37:16.942869+0000","last_clean":"2023-10-09T06:37:16.942869+0000","last_became_active":"2023-10-09T06:33:33.320042+0000","last_became_peered":"2023-10-09T06:33:33.320042+0000","last_unstale":"2023-10-09T06:37:16.942869+0000","last_undegraded":"2023-10-09T06:37:16.942869+0000","last_fullsized":"2023-10-09T06:37:16.942869+0000","mapping_epoch":192,"log_start":"197'3700","ondisk_log_start":"197'3700","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8110,"log_dups_size":0,"ondisk_log_size":8110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:57:35.099867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131596288,"num_objects":2008,"num_object_clones":0,"num_object_copies":6024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11810,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":182,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.334354+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:36:56.334354+0000","last_peered":"2023-10-09T06:36:56.334354+0000","last_clean":"2023-10-09T06:36:56.334354+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:36:56.334354+0000","last_undegraded":"2023-10-09T06:36:56.334354+0000","last_fullsized":"2023-10-09T06:36:56.334354+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993113+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:36:31.993113+0000","last_peered":"2023-10-09T06:36:31.993113+0000","last_clean":"2023-10-09T06:36:31.993113+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:36:31.993113+0000","last_undegraded":"2023-10-09T06:36:31.993113+0000","last_fullsized":"2023-10-09T06:36:31.993113+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"205'11798","reported_seq":11827,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.067774+0000","last_change":"2023-10-09T06:35:45.672810+0000","last_active":"2023-10-09T06:37:19.067774+0000","last_peered":"2023-10-09T06:37:19.067774+0000","last_clean":"2023-10-09T06:37:19.067774+0000","last_became_active":"2023-10-09T06:35:45.654041+0000","last_became_peered":"2023-10-09T06:35:45.654041+0000","last_unstale":"2023-10-09T06:37:19.067774+0000","last_undegraded":"2023-10-09T06:37:19.067774+0000","last_fullsized":"2023-10-09T06:37:19.067774+0000","mapping_epoch":201,"log_start":"197'2400","ondisk_log_start":"197'2400","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9398,"log_dups_size":0,"ondisk_log_size":9398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:59.364217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123469824,"num_objects":1884,"num_object_clones":0,"num_object_copies":5652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11798,"num_write_kb":437824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":135,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993073+0000","last_change":"2023-10-09T06:28:31.028285+0000","last_active":"2023-10-09T06:36:31.993073+0000","last_peered":"2023-10-09T06:36:31.993073+0000","last_clean":"2023-10-09T06:36:31.993073+0000","last_became_active":"2023-10-09T06:28:31.027204+0000","last_became_peered":"2023-10-09T06:28:31.027204+0000","last_unstale":"2023-10-09T06:36:31.993073+0000","last_undegraded":"2023-10-09T06:36:31.993073+0000","last_fullsized":"2023-10-09T06:36:31.993073+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993222+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:36:31.993222+0000","last_peered":"2023-10-09T06:36:31.993222+0000","last_clean":"2023-10-09T06:36:31.993222+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:36:31.993222+0000","last_undegraded":"2023-10-09T06:36:31.993222+0000","last_fullsized":"2023-10-09T06:36:31.993222+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"205'11885","reported_seq":11906,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.801734+0000","last_change":"2023-10-09T06:33:33.244614+0000","last_active":"2023-10-09T06:37:19.801734+0000","last_peered":"2023-10-09T06:37:19.801734+0000","last_clean":"2023-10-09T06:37:19.801734+0000","last_became_active":"2023-10-09T06:33:33.243719+0000","last_became_peered":"2023-10-09T06:33:33.243719+0000","last_unstale":"2023-10-09T06:37:19.801734+0000","last_undegraded":"2023-10-09T06:37:19.801734+0000","last_fullsized":"2023-10-09T06:37:19.801734+0000","mapping_epoch":192,"log_start":"197'3900","ondisk_log_start":"197'3900","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125108224,"num_objects":1909,"num_object_clones":0,"num_object_copies":5727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11885,"num_write_kb":441408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992870+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:36:31.992870+0000","last_peered":"2023-10-09T06:36:31.992870+0000","last_clean":"2023-10-09T06:36:31.992870+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:36:31.992870+0000","last_undegraded":"2023-10-09T06:36:31.992870+0000","last_fullsized":"2023-10-09T06:36:31.992870+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.996873+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:36:31.996873+0000","last_peered":"2023-10-09T06:36:31.996873+0000","last_clean":"2023-10-09T06:36:31.996873+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:36:31.996873+0000","last_undegraded":"2023-10-09T06:36:31.996873+0000","last_fullsized":"2023-10-09T06:36:31.996873+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"205'11607","reported_seq":11637,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.848990+0000","last_change":"2023-10-09T06:35:45.697874+0000","last_active":"2023-10-09T06:37:19.848990+0000","last_peered":"2023-10-09T06:37:19.848990+0000","last_clean":"2023-10-09T06:37:19.848990+0000","last_became_active":"2023-10-09T06:35:45.668802+0000","last_became_peered":"2023-10-09T06:35:45.668802+0000","last_unstale":"2023-10-09T06:37:19.848990+0000","last_undegraded":"2023-10-09T06:37:19.848990+0000","last_fullsized":"2023-10-09T06:37:19.848990+0000","mapping_epoch":201,"log_start":"197'1600","ondisk_log_start":"197'1600","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123404288,"num_objects":1883,"num_object_clones":0,"num_object_copies":5649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11607,"num_write_kb":431680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.983326+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:36:31.983326+0000","last_peered":"2023-10-09T06:36:31.983326+0000","last_clean":"2023-10-09T06:36:31.983326+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:36:31.983326+0000","last_undegraded":"2023-10-09T06:36:31.983326+0000","last_fullsized":"2023-10-09T06:36:31.983326+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.989442+0000","last_change":"2023-10-09T06:35:45.679016+0000","last_active":"2023-10-09T06:36:31.989442+0000","last_peered":"2023-10-09T06:36:31.989442+0000","last_clean":"2023-10-09T06:36:31.989442+0000","last_became_active":"2023-10-09T06:35:45.678002+0000","last_became_peered":"2023-10-09T06:35:45.678002+0000","last_unstale":"2023-10-09T06:36:31.989442+0000","last_undegraded":"2023-10-09T06:36:31.989442+0000","last_fullsized":"2023-10-09T06:36:31.989442+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:22.262853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.e","version":"205'11683","reported_seq":11706,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.028015+0000","last_change":"2023-10-09T06:33:33.317867+0000","last_active":"2023-10-09T06:37:19.028015+0000","last_peered":"2023-10-09T06:37:19.028015+0000","last_clean":"2023-10-09T06:37:19.028015+0000","last_became_active":"2023-10-09T06:33:33.317504+0000","last_became_peered":"2023-10-09T06:33:33.317504+0000","last_unstale":"2023-10-09T06:37:19.028015+0000","last_undegraded":"2023-10-09T06:37:19.028015+0000","last_fullsized":"2023-10-09T06:37:19.028015+0000","mapping_epoch":192,"log_start":"197'2300","ondisk_log_start":"197'2300","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9383,"log_dups_size":0,"ondisk_log_size":9383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:56.906981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121044992,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11683,"num_write_kb":432960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":829,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984033+0000","last_change":"2023-10-09T06:35:45.665207+0000","last_active":"2023-10-09T06:36:31.984033+0000","last_peered":"2023-10-09T06:36:31.984033+0000","last_clean":"2023-10-09T06:36:31.984033+0000","last_became_active":"2023-10-09T06:35:45.664560+0000","last_became_peered":"2023-10-09T06:35:45.664560+0000","last_unstale":"2023-10-09T06:36:31.984033+0000","last_undegraded":"2023-10-09T06:36:31.984033+0000","last_fullsized":"2023-10-09T06:36:31.984033+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:50.880588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994209+0000","last_change":"2023-10-09T06:14:42.690962+0000","last_active":"2023-10-09T06:36:31.994209+0000","last_peered":"2023-10-09T06:36:31.994209+0000","last_clean":"2023-10-09T06:36:31.994209+0000","last_became_active":"2023-10-09T06:14:42.690659+0000","last_became_peered":"2023-10-09T06:14:42.690659+0000","last_unstale":"2023-10-09T06:36:31.994209+0000","last_undegraded":"2023-10-09T06:36:31.994209+0000","last_fullsized":"2023-10-09T06:36:31.994209+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":29,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993803+0000","last_change":"2023-10-09T06:35:45.672632+0000","last_active":"2023-10-09T06:36:31.993803+0000","last_peered":"2023-10-09T06:36:31.993803+0000","last_clean":"2023-10-09T06:36:31.993803+0000","last_became_active":"2023-10-09T06:35:45.672196+0000","last_became_peered":"2023-10-09T06:35:45.672196+0000","last_unstale":"2023-10-09T06:36:31.993803+0000","last_undegraded":"2023-10-09T06:36:31.993803+0000","last_fullsized":"2023-10-09T06:36:31.993803+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"205'11616","reported_seq":11637,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.893307+0000","last_change":"2023-10-09T06:33:33.593915+0000","last_active":"2023-10-09T06:37:19.893307+0000","last_peered":"2023-10-09T06:37:19.893307+0000","last_clean":"2023-10-09T06:37:19.893307+0000","last_became_active":"2023-10-09T06:33:33.593144+0000","last_became_peered":"2023-10-09T06:33:33.593144+0000","last_unstale":"2023-10-09T06:37:19.893307+0000","last_undegraded":"2023-10-09T06:37:19.893307+0000","last_fullsized":"2023-10-09T06:37:19.893307+0000","mapping_epoch":192,"log_start":"197'1600","ondisk_log_start":"197'1600","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11616,"num_write_kb":432768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":843,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993914+0000","last_change":"2023-10-09T06:28:31.472363+0000","last_active":"2023-10-09T06:36:31.993914+0000","last_peered":"2023-10-09T06:36:31.993914+0000","last_clean":"2023-10-09T06:36:31.993914+0000","last_became_active":"2023-10-09T06:28:31.033490+0000","last_became_peered":"2023-10-09T06:28:31.033490+0000","last_unstale":"2023-10-09T06:36:31.993914+0000","last_undegraded":"2023-10-09T06:36:31.993914+0000","last_fullsized":"2023-10-09T06:36:31.993914+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":949,"num_read_kb":2266,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993397+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:36:31.993397+0000","last_peered":"2023-10-09T06:36:31.993397+0000","last_clean":"2023-10-09T06:36:31.993397+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:36:31.993397+0000","last_undegraded":"2023-10-09T06:36:31.993397+0000","last_fullsized":"2023-10-09T06:36:31.993397+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992113+0000","last_change":"2023-10-09T06:35:45.655403+0000","last_active":"2023-10-09T06:36:31.992113+0000","last_peered":"2023-10-09T06:36:31.992113+0000","last_clean":"2023-10-09T06:36:31.992113+0000","last_became_active":"2023-10-09T06:35:45.655106+0000","last_became_peered":"2023-10-09T06:35:45.655106+0000","last_unstale":"2023-10-09T06:36:31.992113+0000","last_undegraded":"2023-10-09T06:36:31.992113+0000","last_fullsized":"2023-10-09T06:36:31.992113+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:20.470509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"205'11556","reported_seq":11579,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.031094+0000","last_change":"2023-10-09T06:33:33.319622+0000","last_active":"2023-10-09T06:37:17.031094+0000","last_peered":"2023-10-09T06:37:17.031094+0000","last_clean":"2023-10-09T06:37:17.031094+0000","last_became_active":"2023-10-09T06:33:33.319140+0000","last_became_peered":"2023-10-09T06:33:33.319140+0000","last_unstale":"2023-10-09T06:37:17.031094+0000","last_undegraded":"2023-10-09T06:37:17.031094+0000","last_fullsized":"2023-10-09T06:37:17.031094+0000","mapping_epoch":192,"log_start":"197'3400","ondisk_log_start":"197'3400","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:00.867299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11556,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":571,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.339135+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:36:56.339135+0000","last_peered":"2023-10-09T06:36:56.339135+0000","last_clean":"2023-10-09T06:36:56.339135+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:36:56.339135+0000","last_undegraded":"2023-10-09T06:36:56.339135+0000","last_fullsized":"2023-10-09T06:36:56.339135+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.990552+0000","last_change":"2023-10-09T06:35:45.655401+0000","last_active":"2023-10-09T06:36:31.990552+0000","last_peered":"2023-10-09T06:36:31.990552+0000","last_clean":"2023-10-09T06:36:31.990552+0000","last_became_active":"2023-10-09T06:35:45.654688+0000","last_became_peered":"2023-10-09T06:35:45.654688+0000","last_unstale":"2023-10-09T06:36:31.990552+0000","last_undegraded":"2023-10-09T06:36:31.990552+0000","last_fullsized":"2023-10-09T06:36:31.990552+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:17.726367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994307+0000","last_change":"2023-10-09T06:35:45.669132+0000","last_active":"2023-10-09T06:36:31.994307+0000","last_peered":"2023-10-09T06:36:31.994307+0000","last_clean":"2023-10-09T06:36:31.994307+0000","last_became_active":"2023-10-09T06:35:45.668751+0000","last_became_peered":"2023-10-09T06:35:45.668751+0000","last_unstale":"2023-10-09T06:36:31.994307+0000","last_undegraded":"2023-10-09T06:36:31.994307+0000","last_fullsized":"2023-10-09T06:36:31.994307+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.9","version":"205'11847","reported_seq":11868,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.859914+0000","last_change":"2023-10-09T06:33:33.244731+0000","last_active":"2023-10-09T06:37:19.859914+0000","last_peered":"2023-10-09T06:37:19.859914+0000","last_clean":"2023-10-09T06:37:19.859914+0000","last_became_active":"2023-10-09T06:33:33.243726+0000","last_became_peered":"2023-10-09T06:33:33.243726+0000","last_unstale":"2023-10-09T06:37:19.859914+0000","last_undegraded":"2023-10-09T06:37:19.859914+0000","last_fullsized":"2023-10-09T06:37:19.859914+0000","mapping_epoch":192,"log_start":"197'1800","ondisk_log_start":"197'1800","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10047,"log_dups_size":0,"ondisk_log_size":10047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119472128,"num_objects":1823,"num_object_clones":0,"num_object_copies":5469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11847,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":553,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994080+0000","last_change":"2023-10-09T06:19:02.848730+0000","last_active":"2023-10-09T06:36:31.994080+0000","last_peered":"2023-10-09T06:36:31.994080+0000","last_clean":"2023-10-09T06:36:31.994080+0000","last_became_active":"2023-10-09T06:19:02.848099+0000","last_became_peered":"2023-10-09T06:19:02.848099+0000","last_unstale":"2023-10-09T06:36:31.994080+0000","last_undegraded":"2023-10-09T06:36:31.994080+0000","last_fullsized":"2023-10-09T06:36:31.994080+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":447,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.002390+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:36:32.002390+0000","last_peered":"2023-10-09T06:36:32.002390+0000","last_clean":"2023-10-09T06:36:32.002390+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:36:32.002390+0000","last_undegraded":"2023-10-09T06:36:32.002390+0000","last_fullsized":"2023-10-09T06:36:32.002390+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":192,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.982210+0000","last_change":"2023-10-09T06:28:31.015024+0000","last_active":"2023-10-09T06:36:31.982210+0000","last_peered":"2023-10-09T06:36:31.982210+0000","last_clean":"2023-10-09T06:36:31.982210+0000","last_became_active":"2023-10-09T06:28:31.014665+0000","last_became_peered":"2023-10-09T06:28:31.014665+0000","last_unstale":"2023-10-09T06:36:31.982210+0000","last_undegraded":"2023-10-09T06:36:31.982210+0000","last_fullsized":"2023-10-09T06:36:31.982210+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:15.980178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":160,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993334+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:36:31.993334+0000","last_peered":"2023-10-09T06:36:31.993334+0000","last_clean":"2023-10-09T06:36:31.993334+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:36:31.993334+0000","last_undegraded":"2023-10-09T06:36:31.993334+0000","last_fullsized":"2023-10-09T06:36:31.993334+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"205'11681","reported_seq":11717,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.416588+0000","last_change":"2023-10-09T06:35:45.735978+0000","last_active":"2023-10-09T06:37:16.416588+0000","last_peered":"2023-10-09T06:37:16.416588+0000","last_clean":"2023-10-09T06:37:16.416588+0000","last_became_active":"2023-10-09T06:35:45.673168+0000","last_became_peered":"2023-10-09T06:35:45.673168+0000","last_unstale":"2023-10-09T06:37:16.416588+0000","last_undegraded":"2023-10-09T06:37:16.416588+0000","last_fullsized":"2023-10-09T06:37:16.416588+0000","mapping_epoch":201,"log_start":"197'1600","ondisk_log_start":"197'1600","created":192,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10081,"log_dups_size":0,"ondisk_log_size":10081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:06.690354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134152192,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11681,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":126,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.002149+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:36:32.002149+0000","last_peered":"2023-10-09T06:36:32.002149+0000","last_clean":"2023-10-09T06:36:32.002149+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:36:32.002149+0000","last_undegraded":"2023-10-09T06:36:32.002149+0000","last_fullsized":"2023-10-09T06:36:32.002149+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.986711+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:36:31.986711+0000","last_peered":"2023-10-09T06:36:31.986711+0000","last_clean":"2023-10-09T06:36:31.986711+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:36:31.986711+0000","last_undegraded":"2023-10-09T06:36:31.986711+0000","last_fullsized":"2023-10-09T06:36:31.986711+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.322293+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:36:56.322293+0000","last_peered":"2023-10-09T06:36:56.322293+0000","last_clean":"2023-10-09T06:36:56.322293+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:36:56.322293+0000","last_undegraded":"2023-10-09T06:36:56.322293+0000","last_fullsized":"2023-10-09T06:36:56.322293+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"205'11515","reported_seq":11538,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.996832+0000","last_change":"2023-10-09T06:33:33.320380+0000","last_active":"2023-10-09T06:37:16.996832+0000","last_peered":"2023-10-09T06:37:16.996832+0000","last_clean":"2023-10-09T06:37:16.996832+0000","last_became_active":"2023-10-09T06:33:33.319960+0000","last_became_peered":"2023-10-09T06:33:33.319960+0000","last_unstale":"2023-10-09T06:37:16.996832+0000","last_undegraded":"2023-10-09T06:37:16.996832+0000","last_fullsized":"2023-10-09T06:37:16.996832+0000","mapping_epoch":192,"log_start":"197'3400","ondisk_log_start":"197'3400","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:25.704605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126812160,"num_objects":1935,"num_object_clones":0,"num_object_copies":5805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11515,"num_write_kb":430400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":115,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:32.003249+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:36:32.003249+0000","last_peered":"2023-10-09T06:36:32.003249+0000","last_clean":"2023-10-09T06:36:32.003249+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:36:32.003249+0000","last_undegraded":"2023-10-09T06:36:32.003249+0000","last_fullsized":"2023-10-09T06:36:32.003249+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992192+0000","last_change":"2023-10-09T06:35:45.662792+0000","last_active":"2023-10-09T06:36:31.992192+0000","last_peered":"2023-10-09T06:36:31.992192+0000","last_clean":"2023-10-09T06:36:31.992192+0000","last_became_active":"2023-10-09T06:35:45.662459+0000","last_became_peered":"2023-10-09T06:35:45.662459+0000","last_unstale":"2023-10-09T06:36:31.992192+0000","last_undegraded":"2023-10-09T06:36:31.992192+0000","last_fullsized":"2023-10-09T06:36:31.992192+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:05.231805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":175,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984254+0000","last_change":"2023-10-09T06:18:59.793139+0000","last_active":"2023-10-09T06:36:31.984254+0000","last_peered":"2023-10-09T06:36:31.984254+0000","last_clean":"2023-10-09T06:36:31.984254+0000","last_became_active":"2023-10-09T06:18:59.792762+0000","last_became_peered":"2023-10-09T06:18:59.792762+0000","last_unstale":"2023-10-09T06:36:31.984254+0000","last_undegraded":"2023-10-09T06:36:31.984254+0000","last_fullsized":"2023-10-09T06:36:31.984254+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:22.503488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"205'12040","reported_seq":12063,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.033373+0000","last_change":"2023-10-09T06:33:33.586554+0000","last_active":"2023-10-09T06:37:19.033373+0000","last_peered":"2023-10-09T06:37:19.033373+0000","last_clean":"2023-10-09T06:37:19.033373+0000","last_became_active":"2023-10-09T06:33:33.586066+0000","last_became_peered":"2023-10-09T06:33:33.586066+0000","last_unstale":"2023-10-09T06:37:19.033373+0000","last_undegraded":"2023-10-09T06:37:19.033373+0000","last_fullsized":"2023-10-09T06:37:19.033373+0000","mapping_epoch":192,"log_start":"197'2600","ondisk_log_start":"197'2600","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":9440,"log_dups_size":0,"ondisk_log_size":9440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:30.973752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127270912,"num_objects":1942,"num_object_clones":0,"num_object_copies":5826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12040,"num_write_kb":447424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":208,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.989487+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:36:31.989487+0000","last_peered":"2023-10-09T06:36:31.989487+0000","last_clean":"2023-10-09T06:36:31.989487+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:36:31.989487+0000","last_undegraded":"2023-10-09T06:36:31.989487+0000","last_fullsized":"2023-10-09T06:36:31.989487+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":389,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.996785+0000","last_change":"2023-10-09T06:35:45.668825+0000","last_active":"2023-10-09T06:36:31.996785+0000","last_peered":"2023-10-09T06:36:31.996785+0000","last_clean":"2023-10-09T06:36:31.996785+0000","last_became_active":"2023-10-09T06:35:45.668234+0000","last_became_peered":"2023-10-09T06:35:45.668234+0000","last_unstale":"2023-10-09T06:36:31.996785+0000","last_undegraded":"2023-10-09T06:36:31.996785+0000","last_fullsized":"2023-10-09T06:36:31.996785+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":207,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.019451+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:36:47.019451+0000","last_peered":"2023-10-09T06:36:47.019451+0000","last_clean":"2023-10-09T06:36:47.019451+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:36:47.019451+0000","last_undegraded":"2023-10-09T06:36:47.019451+0000","last_fullsized":"2023-10-09T06:36:47.019451+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"205'11607","reported_seq":11629,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:37:19.876087+0000","last_change":"2023-10-09T06:33:33.594495+0000","last_active":"2023-10-09T06:37:19.876087+0000","last_peered":"2023-10-09T06:37:19.876087+0000","last_clean":"2023-10-09T06:37:19.876087+0000","last_became_active":"2023-10-09T06:33:33.593206+0000","last_became_peered":"2023-10-09T06:33:33.593206+0000","last_unstale":"2023-10-09T06:37:19.876087+0000","last_undegraded":"2023-10-09T06:37:19.876087+0000","last_fullsized":"2023-10-09T06:37:19.876087+0000","mapping_epoch":192,"log_start":"197'1600","ondisk_log_start":"197'1600","created":192,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:33:32.159382+0000","last_clean_scrub_stamp":"2023-10-09T06:33:32.159382+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122945560,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11607,"num_write_kb":431425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":725,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984580+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:36:31.984580+0000","last_peered":"2023-10-09T06:36:31.984580+0000","last_clean":"2023-10-09T06:36:31.984580+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:36:31.984580+0000","last_undegraded":"2023-10-09T06:36:31.984580+0000","last_fullsized":"2023-10-09T06:36:31.984580+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":167,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T06:36:56.322259+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:36:56.322259+0000","last_peered":"2023-10-09T06:36:56.322259+0000","last_clean":"2023-10-09T06:36:56.322259+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:36:56.322259+0000","last_undegraded":"2023-10-09T06:36:56.322259+0000","last_fullsized":"2023-10-09T06:36:56.322259+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992871+0000","last_change":"2023-10-09T06:28:31.028369+0000","last_active":"2023-10-09T06:36:31.992871+0000","last_peered":"2023-10-09T06:36:31.992871+0000","last_clean":"2023-10-09T06:36:31.992871+0000","last_became_active":"2023-10-09T06:28:31.027228+0000","last_became_peered":"2023-10-09T06:28:31.027228+0000","last_unstale":"2023-10-09T06:36:31.992871+0000","last_undegraded":"2023-10-09T06:36:31.992871+0000","last_fullsized":"2023-10-09T06:36:31.992871+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":168,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984177+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:36:31.984177+0000","last_peered":"2023-10-09T06:36:31.984177+0000","last_clean":"2023-10-09T06:36:31.984177+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:36:31.984177+0000","last_undegraded":"2023-10-09T06:36:31.984177+0000","last_fullsized":"2023-10-09T06:36:31.984177+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.375448+0000","last_change":"2023-10-09T06:14:42.682776+0000","last_active":"2023-10-09T06:36:41.375448+0000","last_peered":"2023-10-09T06:36:41.375448+0000","last_clean":"2023-10-09T06:36:41.375448+0000","last_became_active":"2023-10-09T06:14:42.682423+0000","last_became_peered":"2023-10-09T06:14:42.682423+0000","last_unstale":"2023-10-09T06:36:41.375448+0000","last_undegraded":"2023-10-09T06:36:41.375448+0000","last_fullsized":"2023-10-09T06:36:41.375448+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:15.092504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994220+0000","last_change":"2023-10-09T06:13:43.434610+0000","last_active":"2023-10-09T06:36:31.994220+0000","last_peered":"2023-10-09T06:36:31.994220+0000","last_clean":"2023-10-09T06:36:31.994220+0000","last_became_active":"2023-10-09T06:13:43.434231+0000","last_became_peered":"2023-10-09T06:13:43.434231+0000","last_unstale":"2023-10-09T06:36:31.994220+0000","last_undegraded":"2023-10-09T06:36:31.994220+0000","last_fullsized":"2023-10-09T06:36:31.994220+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993485+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:36:31.993485+0000","last_peered":"2023-10-09T06:36:31.993485+0000","last_clean":"2023-10-09T06:36:31.993485+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:36:31.993485+0000","last_undegraded":"2023-10-09T06:36:31.993485+0000","last_fullsized":"2023-10-09T06:36:31.993485+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.994290+0000","last_change":"2023-10-09T06:28:31.083696+0000","last_active":"2023-10-09T06:36:31.994290+0000","last_peered":"2023-10-09T06:36:31.994290+0000","last_clean":"2023-10-09T06:36:31.994290+0000","last_became_active":"2023-10-09T06:28:31.027818+0000","last_became_peered":"2023-10-09T06:28:31.027818+0000","last_unstale":"2023-10-09T06:36:31.994290+0000","last_undegraded":"2023-10-09T06:36:31.994290+0000","last_fullsized":"2023-10-09T06:36:31.994290+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.993362+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:36:31.993362+0000","last_peered":"2023-10-09T06:36:31.993362+0000","last_clean":"2023-10-09T06:36:31.993362+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:36:31.993362+0000","last_undegraded":"2023-10-09T06:36:31.993362+0000","last_fullsized":"2023-10-09T06:36:31.993362+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":208,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.019525+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:36:47.019525+0000","last_peered":"2023-10-09T06:36:47.019525+0000","last_clean":"2023-10-09T06:36:47.019525+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:36:47.019525+0000","last_undegraded":"2023-10-09T06:36:47.019525+0000","last_fullsized":"2023-10-09T06:36:47.019525+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":201,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.982332+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:36:31.982332+0000","last_peered":"2023-10-09T06:36:31.982332+0000","last_clean":"2023-10-09T06:36:31.982332+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:36:31.982332+0000","last_undegraded":"2023-10-09T06:36:31.982332+0000","last_fullsized":"2023-10-09T06:36:31.982332+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.984621+0000","last_change":"2023-10-09T06:14:12.889605+0000","last_active":"2023-10-09T06:36:31.984621+0000","last_peered":"2023-10-09T06:36:31.984621+0000","last_clean":"2023-10-09T06:36:31.984621+0000","last_became_active":"2023-10-09T06:14:12.889193+0000","last_became_peered":"2023-10-09T06:14:12.889193+0000","last_unstale":"2023-10-09T06:36:31.984621+0000","last_undegraded":"2023-10-09T06:36:31.984621+0000","last_fullsized":"2023-10-09T06:36:31.984621+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:17.304927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":187,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:36:31.992858+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:36:31.992858+0000","last_peered":"2023-10-09T06:36:31.992858+0000","last_clean":"2023-10-09T06:36:31.992858+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:36:31.992858+0000","last_undegraded":"2023-10-09T06:36:31.992858+0000","last_fullsized":"2023-10-09T06:36:31.992858+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":19292,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2013265944,"num_objects":30721,"num_object_clones":0,"num_object_copies":92163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187411,"num_write_kb":6980161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6050230272,"data_stored":6050218056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147911,"ondisk_log_size":147911,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13248,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430672,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920808,"kb_used_data":611296,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92824344,"statfs":{"total":95995035648,"available":95052128256,"internally_reserved":0,"allocated":625967104,"data_stored":625402629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.1859999999999999,"15min":2.5430000000000001},"min":{"1min":0.28199999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.5039999999999996,"5min":5.4699999999999998,"15min":178.738},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.873,"5min":1.012,"15min":2.4300000000000002},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":3.0569999999999999,"5min":4.391,"15min":177.75},"last":1.145}]},{"osd":1,"last update":"Mon Oct 9 06:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.8610000000000002,"15min":2.0609999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.247},"max":{"1min":5.4480000000000004,"5min":148.88499999999999,"15min":148.88499999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.464,"5min":2.7770000000000001,"15min":2.0430000000000001},"min":{"1min":0.32100000000000001,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":13.922000000000001,"5min":148.059,"15min":148.059},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:36:45 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":1.6279999999999999,"15min":1.9199999999999999},"min":{"1min":0.34799999999999998,"5min":0.30499999999999999,"15min":0.29399999999999998},"max":{"1min":15.957000000000001,"5min":15.957000000000001,"15min":61.822000000000003},"last":5.343},{"interface":"front","average":{"1min":1.5489999999999999,"5min":1.296,"15min":1.673},"min":{"1min":0.35699999999999998,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":5.4130000000000003,"5min":5.6269999999999998,"15min":65.064999999999998},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:27 2023","interfaces":[{"interface":"back","average":{"1min":3.419,"5min":1.8340000000000001,"15min":3.7469999999999999},"min":{"1min":0.42799999999999999,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":16.088999999999999,"5min":16.088999999999999,"15min":183.971},"last":5.3879999999999999},{"interface":"front","average":{"1min":2.008,"5min":1.4219999999999999,"15min":3.4319999999999999},"min":{"1min":0.439,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.0949999999999998,"5min":7.3019999999999996,"15min":183.095},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.3260000000000001,"15min":1.1739999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":8.5419999999999998,"5min":12.382999999999999,"15min":12.382999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.736,"15min":1.6419999999999999},"min":{"1min":0.39800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.742,"5min":16.064,"15min":16.064},"last":5.7279999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.2889999999999999,"15min":1.254},"min":{"1min":0.373,"5min":0.34799999999999998,"15min":0.28199999999999997},"max":{"1min":10.680999999999999,"5min":10.680999999999999,"15min":10.680999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.4300000000000002,"5min":1.821,"15min":1.4159999999999999},"min":{"1min":0.46300000000000002,"5min":0.33900000000000002,"15min":0.33100000000000002},"max":{"1min":11.444000000000001,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.78400000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.353,"15min":1.2709999999999999},"min":{"1min":0.40799999999999997,"5min":0.314,"15min":0.314},"max":{"1min":3.9319999999999999,"5min":8.2439999999999998,"15min":15.521000000000001},"last":0.435},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.746,"15min":1.583},"min":{"1min":0.378,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":7.9420000000000002,"5min":16.024000000000001,"15min":16.024000000000001},"last":11.747999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626904,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910036,"kb_used_data":608716,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92835116,"statfs":{"total":95995035648,"available":95063158784,"internally_reserved":0,"allocated":623325184,"data_stored":622780408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.415,"15min":1.248},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.6280000000000001,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.245},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.534,"15min":1.4379999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.26300000000000001},"max":{"1min":5.3789999999999996,"5min":10.696,"15min":10.696},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":4.2249999999999996,"15min":2.7799999999999998},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":4.8769999999999998,"5min":316.88099999999997,"15min":316.88099999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.492,"5min":4.1260000000000003,"15min":2.7370000000000001},"min":{"1min":0.30299999999999999,"5min":0.29799999999999999,"15min":0.27300000000000002},"max":{"1min":4.2809999999999997,"5min":316.84399999999999,"15min":316.84399999999999},"last":5.2809999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.8080000000000001,"15min":1.8959999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.5170000000000003,"5min":26.477,"15min":53.616},"last":5.4550000000000001},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.7689999999999999,"15min":1.901},"min":{"1min":0.376,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":4.5579999999999998,"5min":25.745999999999999,"15min":53.665999999999997},"last":5.0060000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:37:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.0150000000000001,"15min":2.4860000000000002},"min":{"1min":0.32800000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":8.7210000000000001,"5min":27.491,"15min":115.09399999999999},"last":4.6559999999999997},{"interface":"front","average":{"1min":2.4289999999999998,"5min":2.02,"15min":2.2280000000000002},"min":{"1min":0.45500000000000002,"5min":0.36799999999999999,"15min":0.32000000000000001},"max":{"1min":8.0340000000000007,"5min":26.815000000000001,"15min":112.886},"last":1.8400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.2649999999999999,"15min":1.27},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26000000000000001},"max":{"1min":8.0500000000000007,"5min":10.653,"15min":10.653},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.427,"15min":1.4650000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":5.1440000000000001,"5min":11.644,"15min":11.644},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:36:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.2410000000000001,"15min":1.425},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34300000000000003},"max":{"1min":8.0939999999999994,"5min":10.601000000000001,"15min":11.754},"last":6.0129999999999999},{"interface":"front","average":{"1min":2.528,"5min":1.8520000000000001,"15min":1.637},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2040000000000006,"5min":9.2040000000000006,"15min":12.295999999999999},"last":11.526}]},{"osd":7,"last update":"Mon Oct 9 06:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.2769999999999999,"15min":1.397},"min":{"1min":0.42199999999999999,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":8.1219999999999999,"5min":10.673,"15min":11.076000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.2450000000000001,"15min":1.2829999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.968,"5min":6.077,"15min":11.102},"last":5.577}]}]},{"osd":1,"up_from":93,"seq":399431958813,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1319340,"kb_used_data":894244,"kb_used_omap":7,"kb_used_meta":425080,"kb_avail":92425812,"statfs":{"total":95995035648,"available":94644031488,"internally_reserved":0,"allocated":915705856,"data_stored":914848002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":435282320},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":3.7850000000000001,"15min":2.5499999999999998},"min":{"1min":0.30299999999999999,"5min":0.216,"15min":0.187},"max":{"1min":8.125,"5min":132.886,"15min":132.886},"last":0.318},{"interface":"front","average":{"1min":3.2010000000000001,"5min":4.5250000000000004,"15min":2.8780000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":13.411,"5min":133.315,"15min":133.315},"last":1.6060000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:37:03 2023","interfaces":[{"interface":"back","average":{"1min":3.5600000000000001,"5min":2.0529999999999999,"15min":2.0609999999999999},"min":{"1min":0.23300000000000001,"5min":0.224,"15min":0.19800000000000001},"max":{"1min":9.7840000000000007,"5min":26.210000000000001,"15min":67.206000000000003},"last":16.734000000000002},{"interface":"front","average":{"1min":3.9950000000000001,"5min":2.1749999999999998,"15min":2.1309999999999998},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.224},"max":{"1min":16.302,"5min":26.026,"15min":67.132000000000005},"last":16.654}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":7.258,"5min":4.3449999999999998,"15min":4.516},"min":{"1min":0.27800000000000002,"5min":0.251,"15min":0.251},"max":{"1min":110.711,"5min":132.792,"15min":132.792},"last":16.768000000000001},{"interface":"front","average":{"1min":8.1660000000000004,"5min":4.3840000000000003,"15min":4.5519999999999996},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":111.48999999999999,"5min":132.53299999999999,"15min":132.53299999999999},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.5049999999999999,"15min":1.5149999999999999},"min":{"1min":0.34799999999999998,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":6.2549999999999999,"5min":26.068999999999999,"15min":26.068999999999999},"last":16.617999999999999},{"interface":"front","average":{"1min":1.5720000000000001,"5min":1.2849999999999999,"15min":1.4810000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.31},"max":{"1min":8.2159999999999993,"5min":8.2159999999999993,"15min":12.907999999999999},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.0270000000000001,"15min":1.7230000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.2300000000000004,"5min":10.586,"15min":14.170999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.9339999999999999,"5min":1.639,"15min":1.6379999999999999},"min":{"1min":0.437,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.9219999999999997,"5min":26.157,"15min":26.157},"last":16.702999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":1.9850000000000001,"15min":1.7170000000000001},"min":{"1min":0.379,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.2800000000000002,"5min":10.625,"15min":14.212999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.02,"5min":1.8919999999999999,"15min":1.7310000000000001},"min":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.34999999999999998},"max":{"1min":6.7309999999999999,"5min":26.132000000000001,"15min":26.132000000000001},"last":16.670000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.901,"15min":1.9410000000000001},"min":{"1min":0.47499999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.657,"5min":26.109000000000002,"15min":30.774999999999999},"last":16.687999999999999},{"interface":"front","average":{"1min":2.016,"5min":1.4379999999999999,"15min":1.663},"min":{"1min":0.51700000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":10.025,"5min":10.025,"15min":30.815000000000001},"last":0.499}]}]},{"osd":0,"up_from":88,"seq":377957122339,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265040,"kb_used_data":735752,"kb_used_omap":8,"kb_used_meta":529271,"kb_avail":92480112,"statfs":{"total":95995035648,"available":94699634688,"internally_reserved":0,"allocated":753410048,"data_stored":752577085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":541973760},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":2.4140000000000001,"15min":2.677},"min":{"1min":0.21199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":6.8920000000000003,"5min":25.084,"15min":138.31700000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.153,"15min":2.5059999999999998},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.20200000000000001},"max":{"1min":6.9500000000000002,"5min":25.036999999999999,"15min":144.37},"last":0.54600000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.905,"15min":1.9199999999999999},"min":{"1min":0.24199999999999999,"5min":0.185,"15min":0.185},"max":{"1min":5.7110000000000003,"5min":13.425000000000001,"15min":28.745000000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.738,"15min":1.837},"min":{"1min":0.248,"5min":0.22600000000000001,"15min":0.18099999999999999},"max":{"1min":9.0909999999999993,"5min":16.129000000000001,"15min":28.146999999999998},"last":0.89600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":2.3599999999999999,"15min":2.6219999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":5.3399999999999999,"5min":33.741999999999997,"15min":59.643000000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":1.494,"5min":2.2999999999999998,"15min":2.4929999999999999},"min":{"1min":0.28000000000000003,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":5.4050000000000002,"5min":32.859000000000002,"15min":59.762999999999998},"last":3.0750000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:37:13 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.5429999999999999,"15min":1.4730000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":9.4209999999999994,"5min":9.4209999999999994,"15min":9.4209999999999994},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.411,"5min":1.76,"15min":1.7709999999999999},"min":{"1min":0.371,"5min":0.317,"15min":0.317},"max":{"1min":10.238,"5min":10.238,"15min":28.629000000000001},"last":1.6799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:37:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":2.1000000000000001,"15min":2.0619999999999998},"min":{"1min":0.40000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.151,"5min":11.042999999999999,"15min":28.574000000000002},"last":0.83999999999999997},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.669,"15min":1.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32600000000000001},"max":{"1min":10.419,"5min":10.419,"15min":28.061},"last":0.74299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:37:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6749999999999998,"5min":1.756,"15min":1.8129999999999999},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.377000000000001,"5min":10.377000000000001,"15min":27.966999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.8009999999999999,"5min":1.506,"15min":1.472},"min":{"1min":0.39400000000000002,"5min":0.38900000000000001,"15min":0.33700000000000002},"max":{"1min":9.4510000000000005,"5min":9.4510000000000005,"15min":9.4510000000000005},"last":6.2119999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:37:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7869999999999999,"5min":2.0920000000000001,"15min":2.0099999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.40300000000000002},"max":{"1min":13.259,"5min":13.259,"15min":28.109999999999999},"last":0.98899999999999999},{"interface":"front","average":{"1min":3.5409999999999999,"5min":1.9810000000000001,"15min":1.911},"min":{"1min":0.44500000000000001,"5min":0.40600000000000003,"15min":0.38400000000000001},"max":{"1min":13.428000000000001,"5min":13.428000000000001,"15min":28.074999999999999},"last":0.997}]}]},{"osd":2,"up_from":98,"seq":420906795287,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1302676,"kb_used_data":789068,"kb_used_omap":11,"kb_used_meta":513396,"kb_avail":92442476,"statfs":{"total":95995035648,"available":94661095424,"internally_reserved":0,"allocated":808005632,"data_stored":807165889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":525718462},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.871,"15min":2.5019999999999998},"min":{"1min":0.23499999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":32.603000000000002,"5min":64.356999999999999,"15min":117.2},"last":0.52300000000000002},{"interface":"front","average":{"1min":3.6709999999999998,"5min":2.3580000000000001,"15min":2.3410000000000002},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":32.533999999999999,"5min":63.968000000000004,"15min":121.521},"last":7.6589999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":3.3450000000000002,"15min":2.3039999999999998},"min":{"1min":0.29199999999999998,"5min":0.24099999999999999,"15min":0.17999999999999999},"max":{"1min":6.2140000000000004,"5min":60.155000000000001,"15min":60.155000000000001},"last":3.254},{"interface":"front","average":{"1min":1.764,"5min":2.9199999999999999,"15min":2.1960000000000002},"min":{"1min":0.26900000000000002,"5min":0.26100000000000001,"15min":0.215},"max":{"1min":6.0090000000000003,"5min":62.582999999999998,"15min":62.582999999999998},"last":7.7350000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:36:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.4700000000000002,"15min":3.5640000000000001},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":9.9670000000000005,"5min":27.050999999999998,"15min":201.33600000000001},"last":3.2869999999999999},{"interface":"front","average":{"1min":3.7519999999999998,"5min":2.2269999999999999,"15min":3.6070000000000002},"min":{"1min":0.32600000000000001,"5min":0.28399999999999997,"15min":0.21299999999999999},"max":{"1min":32.167000000000002,"5min":32.167000000000002,"15min":199.95099999999999},"last":3.1259999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5209999999999999,"5min":1.7290000000000001,"15min":1.7869999999999999},"min":{"1min":0.34699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":32.433,"5min":32.433,"15min":32.433},"last":7.5590000000000002},{"interface":"front","average":{"1min":3.5299999999999998,"5min":1.6220000000000001,"15min":1.6950000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34200000000000003},"max":{"1min":32.286999999999999,"5min":32.286999999999999,"15min":32.286999999999999},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.8129999999999999,"15min":2.2839999999999998},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.33900000000000002},"max":{"1min":32.475999999999999,"5min":32.475999999999999,"15min":86.423000000000002},"last":7.6040000000000001},{"interface":"front","average":{"1min":3.4060000000000001,"5min":2.0369999999999999,"15min":2.0979999999999999},"min":{"1min":0.38300000000000001,"5min":0.377,"15min":0.33800000000000002},"max":{"1min":32.331000000000003,"5min":32.331000000000003,"15min":86.382999999999996},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":1.7849999999999999,"15min":1.744},"min":{"1min":0.40799999999999997,"5min":0.38900000000000001,"15min":0.32800000000000001},"max":{"1min":32.357999999999997,"5min":32.357999999999997,"15min":32.357999999999997},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.8279999999999998,"5min":2.077,"15min":1.8069999999999999},"min":{"1min":0.41499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":32.377000000000002,"5min":32.377000000000002,"15min":32.377000000000002},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:37:00 2023","interfaces":[{"interface":"back","average":{"1min":3.335,"5min":2.1070000000000002,"15min":1.8779999999999999},"min":{"1min":0.45000000000000001,"5min":0.40999999999999998,"15min":0.40500000000000003},"max":{"1min":32.417999999999999,"5min":32.417999999999999,"15min":32.417999999999999},"last":1.7470000000000001},{"interface":"front","average":{"1min":3.1819999999999999,"5min":1.8999999999999999,"15min":1.9099999999999999},"min":{"1min":0.434,"5min":0.39300000000000002,"15min":0.35399999999999998},"max":{"1min":32.505000000000003,"5min":32.505000000000003,"15min":32.505000000000003},"last":7.6340000000000003}]}]},{"osd":3,"up_from":103,"seq":442381631761,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1323096,"kb_used_data":783312,"kb_used_omap":3,"kb_used_meta":539772,"kb_avail":92422056,"statfs":{"total":95995035648,"available":94640185344,"internally_reserved":0,"allocated":802111488,"data_stored":801268517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":552727073},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.1059999999999999,"15min":2.52},"min":{"1min":0.26200000000000001,"5min":0.224,"15min":0.17399999999999999},"max":{"1min":6.7720000000000002,"5min":27.911999999999999,"15min":147.988},"last":9.391},{"interface":"front","average":{"1min":2.4009999999999998,"5min":2.2469999999999999,"15min":2.375},"min":{"1min":0.22800000000000001,"5min":0.19,"15min":0.18099999999999999},"max":{"1min":11.661,"5min":26.988,"15min":147.94200000000001},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.319,"15min":1.865},"min":{"1min":0.219,"5min":0.20799999999999999,"15min":0.19},"max":{"1min":6.9829999999999997,"5min":47.057000000000002,"15min":47.057000000000002},"last":0.89500000000000002},{"interface":"front","average":{"1min":1.613,"5min":2.2650000000000001,"15min":2.1379999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.218},"max":{"1min":4.9740000000000002,"5min":54.752000000000002,"15min":54.752000000000002},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.8280000000000001,"15min":2.2709999999999999},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.20599999999999999},"max":{"1min":7.4279999999999999,"5min":14.968,"15min":47.097999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.573,"5min":1.6040000000000001,"15min":1.7969999999999999},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":6.1109999999999998,"5min":11.683999999999999,"15min":46.997},"last":3.7240000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":1.728,"15min":1.9870000000000001},"min":{"1min":0.40500000000000003,"5min":0.315,"15min":0.30399999999999999},"max":{"1min":7.3620000000000001,"5min":8.6470000000000002,"15min":72.653000000000006},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.764,"5min":1.542,"15min":2.0230000000000001},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.048,"5min":11.756,"15min":72.706999999999994},"last":0.60399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.579,"15min":1.7290000000000001},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.9569999999999999,"5min":11.781000000000001,"15min":11.781000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.637,"5min":1.831,"15min":1.867},"min":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.32300000000000001},"max":{"1min":10.315,"5min":10.315,"15min":11.454000000000001},"last":0.66800000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.49,"15min":1.7669999999999999},"min":{"1min":0.42099999999999999,"5min":0.40400000000000003,"15min":0.36099999999999999},"max":{"1min":7.984,"5min":10.384,"15min":15.122999999999999},"last":0.62},{"interface":"front","average":{"1min":1.768,"5min":1.6140000000000001,"15min":1.681},"min":{"1min":0.42299999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.0220000000000002,"5min":11.728999999999999,"15min":11.728999999999999},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8540000000000001,"5min":2.1589999999999998,"15min":2.0070000000000001},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":7.6470000000000002,"5min":8.6059999999999999,"15min":18.695},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.6040000000000001,"15min":1.9590000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.39900000000000002},"max":{"1min":4.9539999999999997,"5min":10.409000000000001,"15min":13.211},"last":1.0589999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":922972,"kb_used_data":609364,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92822180,"statfs":{"total":95995035648,"available":95049912320,"internally_reserved":0,"allocated":623988736,"data_stored":623435911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.6970000000000001,"15min":1.8460000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.5279999999999996,"5min":16.218,"15min":115.59699999999999},"last":24.253},{"interface":"front","average":{"1min":1.363,"5min":1.8400000000000001,"15min":2.1429999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":7.3739999999999997,"5min":15.242000000000001,"15min":115.65000000000001},"last":24.204999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.6319999999999999,"15min":2.1269999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":3.3439999999999999,"5min":17.033000000000001,"15min":179.47300000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.714,"15min":2.4100000000000001},"min":{"1min":0.377,"5min":0.316,"15min":0.28499999999999998},"max":{"1min":3.3319999999999999,"5min":17.015999999999998,"15min":179.447},"last":0.85799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:37:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.885,"15min":1.655},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":14.705,"15min":17.614000000000001},"last":24.068000000000001},{"interface":"front","average":{"1min":1.798,"5min":1.5840000000000001,"15min":1.7070000000000001},"min":{"1min":0.35999999999999999,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":6.0999999999999996,"5min":6.7850000000000001,"15min":17.439},"last":23.434000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.7090000000000001,"15min":2.1659999999999999},"min":{"1min":0.40899999999999997,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":2.9750000000000001,"5min":8.1449999999999996,"15min":91.361999999999995},"last":24.285},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.6679999999999999,"15min":1.8899999999999999},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.313},"max":{"1min":2.9620000000000002,"5min":16.257000000000001,"15min":89.676000000000002},"last":24.109999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:36:36 2023","interfaces":[{"interface":"back","average":{"1min":2.722,"5min":1.522,"15min":1.264},"min":{"1min":0.33500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":22.068000000000001,"5min":22.068000000000001,"15min":22.068000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.831,"15min":1.575},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.26500000000000001},"max":{"1min":4.0170000000000003,"5min":13.095000000000001,"15min":13.095000000000001},"last":0.93799999999999994}]},{"osd":6,"last update":"Mon Oct 9 06:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.617,"15min":1.4119999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.314},"max":{"1min":7.3220000000000001,"5min":7.9020000000000001,"15min":8.1110000000000007},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.25,"5min":1.357,"15min":1.2649999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5590000000000002,"5min":11.782,"15min":11.782},"last":0.83399999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":1.5269999999999999,"15min":1.5760000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.371},"max":{"1min":7.3550000000000004,"5min":7.3550000000000004,"15min":12.779},"last":0.81000000000000005},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.4550000000000001,"15min":1.5269999999999999},"min":{"1min":0.46899999999999997,"5min":0.317,"15min":0.317},"max":{"1min":7.7670000000000003,"5min":9.1370000000000005,"15min":14.896000000000001},"last":0.97899999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823137,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1570892,"kb_used_data":985284,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92174260,"statfs":{"total":95995035648,"available":94386442240,"internally_reserved":0,"allocated":1008930816,"data_stored":1008389532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":2.367,"15min":2.1549999999999998},"min":{"1min":0.27500000000000002,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":9.3350000000000009,"5min":83.197000000000003,"15min":83.197000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":3.3029999999999999,"5min":2.7210000000000001,"15min":2.1309999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":15.114000000000001,"5min":83.563999999999993,"15min":83.563999999999993},"last":0.32300000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:36:39 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":2.3260000000000001,"15min":2.4910000000000001},"min":{"1min":0.32400000000000001,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":5.7930000000000001,"5min":62.494,"15min":161.71199999999999},"last":3.391},{"interface":"front","average":{"1min":2.1509999999999998,"5min":2.5899999999999999,"15min":2.7330000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.9890000000000008,"5min":62.613999999999997,"15min":162.50200000000001},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:36:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":1.6950000000000001,"15min":2.0409999999999999},"min":{"1min":0.46600000000000003,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":10.083,"5min":10.083,"15min":54.981999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.7769999999999999,"15min":1.9830000000000001},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":11.787000000000001,"5min":11.787000000000001,"15min":59.082999999999998},"last":4.8250000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:36:25 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":4.2060000000000004,"15min":3.3580000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":7.1210000000000004,"5min":84.174000000000007,"15min":84.174000000000007},"last":0.441},{"interface":"front","average":{"1min":2.7629999999999999,"5min":4.1710000000000003,"15min":3.2050000000000001},"min":{"1min":0.35099999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":11.747,"5min":83.114999999999995,"15min":83.114999999999995},"last":0.752}]},{"osd":4,"last update":"Mon Oct 9 06:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":1.6160000000000001,"15min":1.671},"min":{"1min":0.375,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":9.0459999999999994,"5min":10.438000000000001,"15min":10.617000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.679,"5min":1.163,"15min":1.3779999999999999},"min":{"1min":0.35099999999999998,"5min":0.308,"15min":0.308},"max":{"1min":6.8499999999999996,"5min":6.8499999999999996,"15min":16.181999999999999},"last":0.5}]},{"osd":6,"last update":"Mon Oct 9 06:36:25 2023","interfaces":[{"interface":"back","average":{"1min":4.1609999999999996,"5min":1.8959999999999999,"15min":1.6930000000000001},"min":{"1min":0.42499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":19.908000000000001,"5min":19.908000000000001,"15min":19.908000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.532,"5min":1.462,"15min":1.536},"min":{"1min":0.441,"5min":0.34899999999999998,"15min":0.33400000000000002},"max":{"1min":11.837,"5min":11.837,"15min":12.365},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:36:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.4970000000000001,"15min":1.502},"min":{"1min":0.47599999999999998,"5min":0.442,"15min":0.36899999999999999},"max":{"1min":9.6319999999999997,"5min":9.6319999999999997,"15min":9.6319999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.649,"5min":1.575,"15min":1.3919999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":11.829000000000001,"5min":11.829000000000001,"15min":11.829000000000001},"last":0.71999999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":745996288,"data_stored":745996288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":896532480,"data_stored":896532480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":786239488,"data_stored":786235416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":782958592,"data_stored":782958592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":618004480,"data_stored":618004480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":988614656,"data_stored":988610584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":609353728,"data_stored":609353728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":622530560,"data_stored":622526488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:37:25.609 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:37:25.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-10-09T06:37:25.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:25.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.821 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:37:25.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:25.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:25.880 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:37:25.939 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:37:25.965 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:37:25.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:26.006 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:37:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:26.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:26.045 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:37:26.090 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:37:26.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:26.139 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:37:26.150 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.150 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.151 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:37:26.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:26.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:26.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:26.322 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:37:26.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:26.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:26 smithi155 bash[59495]: cluster 2023-10-09T06:37:25.200203+0000 mgr.y (mgr.25723) 983 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-09T06:37:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:26 smithi155 bash[59495]: audit 2023-10-09T06:37:25.767529+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.125:0/545000027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:26 smithi155 bash[59495]: audit 2023-10-09T06:37:26.062930+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:26 smithi155 bash[59495]: audit 2023-10-09T06:37:26.064286+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.125:0/3635413941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:26.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:26.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[77299]: cluster 2023-10-09T06:37:25.200203+0000 mgr.y (mgr.25723) 983 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-09T06:37:26.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[77299]: audit 2023-10-09T06:37:25.767529+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.125:0/545000027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:26.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[77299]: audit 2023-10-09T06:37:26.062930+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.536 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[77299]: audit 2023-10-09T06:37:26.064286+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.125:0/3635413941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[80002]: cluster 2023-10-09T06:37:25.200203+0000 mgr.y (mgr.25723) 983 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 1.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-09T06:37:26.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[80002]: audit 2023-10-09T06:37:25.767529+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.125:0/545000027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:26.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[80002]: audit 2023-10-09T06:37:26.062930+0000 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:26 smithi125 bash[80002]: audit 2023-10-09T06:37:26.064286+0000 mon.b (mon.1) 27 : audit 1 from='client.? 172.21.15.125:0/3635413941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:26.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:26.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:26.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:26.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:26.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:26.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:26.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:26.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:27.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.107 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:27.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:27.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.294891+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: audit 2023-10-09T06:37:26.300869+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.300928+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.311281+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.312123+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.313644+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.327 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.321031+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.327 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:27 smithi155 bash[59495]: cluster 2023-10-09T06:37:26.323087+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:27.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:27.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.294891+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:27.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: audit 2023-10-09T06:37:26.300869+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:27.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.300928+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.311281+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.312123+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.313644+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.321031+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[77299]: cluster 2023-10-09T06:37:26.323087+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.294891+0000 mon.a (mon.0) 1176 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: audit 2023-10-09T06:37:26.300869+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.300928+0000 mon.a (mon.0) 1178 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:37:27.482 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.311281+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.312123+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.313644+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.321031+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:27 smithi125 bash[80002]: cluster 2023-10-09T06:37:26.323087+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e206 with expected crc 2023-10-09T06:37:27.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:27.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:27.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:27.646 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:27.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:27.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:27.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:27.928 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:27.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:28.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:28.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:28.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:28.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:28.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:28.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:28.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:28.447 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:28 smithi125 bash[77299]: cluster 2023-10-09T06:37:27.200852+0000 mgr.y (mgr.25723) 984 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:37:28.447 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:28 smithi125 bash[80002]: cluster 2023-10-09T06:37:27.200852+0000 mgr.y (mgr.25723) 984 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:37:28.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:28.580 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:28.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:28 smithi155 bash[59495]: cluster 2023-10-09T06:37:27.200852+0000 mgr.y (mgr.25723) 984 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:37:28.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:28.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:28.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:28.873 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:28.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:28.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:29.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:29.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:29.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:29.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:29.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:29.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:29.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:29.430 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:29.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:29.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:29.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:29.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:29.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:29.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:29.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:29.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:29.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:29.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:30.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:30.126 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:30.141 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:29 smithi125 bash[77299]: cluster 2023-10-09T06:37:29.201372+0000 mgr.y (mgr.25723) 985 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:30.141 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:29 smithi125 bash[80002]: cluster 2023-10-09T06:37:29.201372+0000 mgr.y (mgr.25723) 985 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:30.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:30.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:30.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:30.375 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:29 smithi155 bash[59495]: cluster 2023-10-09T06:37:29.201372+0000 mgr.y (mgr.25723) 985 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-09T06:37:30.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:30.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:30.403 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:30.403 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:30.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:30.528 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:30.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:30.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:30.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:30.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:37:30.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:37:30.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:30.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:30.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:30.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:31.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:31.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:31.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:31.237 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:31] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:37:31.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:31.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:31.478 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:31.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:31.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:31.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:31.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:31.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:37:31.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:31.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:32.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:32.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:32.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:32.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:32.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:32.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:32.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:32.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:32.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:32 smithi125 bash[77299]: cluster 2023-10-09T06:37:31.202060+0000 mgr.y (mgr.25723) 986 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:32.460 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:32 smithi125 bash[80002]: cluster 2023-10-09T06:37:31.202060+0000 mgr.y (mgr.25723) 986 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:32.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:32.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:32.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:32.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:32 smithi155 bash[59495]: cluster 2023-10-09T06:37:31.202060+0000 mgr.y (mgr.25723) 986 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 1.5 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-09T06:37:32.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:32.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:32.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:32.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:32.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:32.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:32.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:33.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:33.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:33.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:33.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:33.292 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:33.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:33.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:33.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:33.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:33.549 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:33.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:33.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:33.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:33.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:33.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:33.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:33.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:33.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:33.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:34.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:34.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:34.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:34.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:34.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:34.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:34 smithi125 bash[77299]: cluster 2023-10-09T06:37:33.202657+0000 mgr.y (mgr.25723) 987 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:37:34.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:34 smithi125 bash[77299]: audit 2023-10-09T06:37:33.594211+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:34.377 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:34 smithi125 bash[80002]: cluster 2023-10-09T06:37:33.202657+0000 mgr.y (mgr.25723) 987 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:37:34.378 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:34 smithi125 bash[80002]: audit 2023-10-09T06:37:33.594211+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:34.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:34.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:34 smithi155 bash[59495]: cluster 2023-10-09T06:37:33.202657+0000 mgr.y (mgr.25723) 987 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-09T06:37:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:34 smithi155 bash[59495]: audit 2023-10-09T06:37:33.594211+0000 mon.a (mon.0) 1179 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:34.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:34.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:34.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:34.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:34.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:35.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:35.102 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:35.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:35.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:35.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:35.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:35.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:35.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:35.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:35.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:35.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:35.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:35.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:35.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:35.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:36.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:36.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:36.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:36.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:36.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:36.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:36.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:36.579 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:36 smithi155 bash[59495]: cluster 2023-10-09T06:37:35.203491+0000 mgr.y (mgr.25723) 988 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:37:36.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:36.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:36 smithi125 bash[77299]: cluster 2023-10-09T06:37:35.203491+0000 mgr.y (mgr.25723) 988 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:37:36.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:36 smithi125 bash[80002]: cluster 2023-10-09T06:37:35.203491+0000 mgr.y (mgr.25723) 988 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-09T06:37:36.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:36.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:36.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.871 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:36.872 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:36.872 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:36.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:36.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:37.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:37.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:37.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:37.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:37.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:37.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:37.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:37.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:37.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:37.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:37.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:37.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:37.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:37.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:37.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:37.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:38.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:38.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:38.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:38.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:38.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:38.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:38.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:38.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:38 smithi125 bash[77299]: cluster 2023-10-09T06:37:37.204152+0000 mgr.y (mgr.25723) 989 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:37:38.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:38 smithi125 bash[80002]: cluster 2023-10-09T06:37:37.204152+0000 mgr.y (mgr.25723) 989 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:37:38.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:38.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:38.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:38.611 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:38 smithi155 bash[59495]: cluster 2023-10-09T06:37:37.204152+0000 mgr.y (mgr.25723) 989 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:37:38.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:38.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:38.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:38.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:38.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:38.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:39.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:39.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:39.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:39.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:39.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:39.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:39.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.609 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:39.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:39.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:39.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:39.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:39.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:40.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:40.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:40.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:40.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:40.267 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:39 smithi155 bash[59495]: cluster 2023-10-09T06:37:39.204721+0000 mgr.y (mgr.25723) 990 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:37:40.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:40.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:39 smithi125 bash[77299]: cluster 2023-10-09T06:37:39.204721+0000 mgr.y (mgr.25723) 990 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:37:40.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:39 smithi125 bash[80002]: cluster 2023-10-09T06:37:39.204721+0000 mgr.y (mgr.25723) 990 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-10-09T06:37:40.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:40.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:40.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:40.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:40.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:40.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:40.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:40.818 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:40.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:40.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:40.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:41.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:41.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:41.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:41.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:41.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:41.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:41.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:41.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:41.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:41.489 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:41] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:37:41.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:41.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:41.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:41.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:41.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:41.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:41.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:41.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:41.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:42.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:42.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:42.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:42.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:42.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:42.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:42 smithi125 bash[80002]: cluster 2023-10-09T06:37:41.205383+0000 mgr.y (mgr.25723) 991 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:37:42.288 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:42 smithi125 bash[80002]: audit 2023-10-09T06:37:41.515817+0000 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:37:42.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:42 smithi125 bash[77299]: cluster 2023-10-09T06:37:41.205383+0000 mgr.y (mgr.25723) 991 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:37:42.288 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:42 smithi125 bash[77299]: audit 2023-10-09T06:37:41.515817+0000 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:37:42.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:42.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:42.293 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 8 pg_num to 26 2023-10-09T06:37:42.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:42.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:42 smithi155 bash[59495]: cluster 2023-10-09T06:37:41.205383+0000 mgr.y (mgr.25723) 991 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-09T06:37:42.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:42 smithi155 bash[59495]: audit 2023-10-09T06:37:41.515817+0000 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:37:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:42.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:42.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:42.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:42.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:42.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:42.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:42.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:42.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:42.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:42.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:42.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:43.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:43.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:43.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:43.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:43.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:43 smithi155 bash[59495]: audit 2023-10-09T06:37:42.273683+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:37:43.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:43 smithi155 bash[59495]: cluster 2023-10-09T06:37:42.273732+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:37:43.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:43 smithi155 bash[59495]: cluster 2023-10-09T06:37:42.279295+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:43 smithi155 bash[59495]: cluster 2023-10-09T06:37:42.299402+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:43 smithi155 bash[59495]: audit 2023-10-09T06:37:43.206961+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:37:43.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:43.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[77299]: audit 2023-10-09T06:37:42.273683+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[77299]: cluster 2023-10-09T06:37:42.273732+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[77299]: cluster 2023-10-09T06:37:42.279295+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[77299]: cluster 2023-10-09T06:37:42.299402+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[77299]: audit 2023-10-09T06:37:43.206961+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:37:43.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[80002]: audit 2023-10-09T06:37:42.273683+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.125:0/2933767214' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:37:43.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[80002]: cluster 2023-10-09T06:37:42.273732+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:37:43.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[80002]: cluster 2023-10-09T06:37:42.279295+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[80002]: cluster 2023-10-09T06:37:42.299402+0000 osd.4 (osd.4) 132 : cluster 3 failed to encode map e207 with expected crc 2023-10-09T06:37:43.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:43 smithi125 bash[80002]: audit 2023-10-09T06:37:43.206961+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:37:43.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:43.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:43.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:43.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.944 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:43.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:44.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:44.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:44.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:44.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: cluster 2023-10-09T06:37:43.205881+0000 mgr.y (mgr.25723) 992 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 1021 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:37:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: audit 2023-10-09T06:37:43.297865+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:37:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: cluster 2023-10-09T06:37:43.297979+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:37:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: cluster 2023-10-09T06:37:43.299328+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: cluster 2023-10-09T06:37:43.319216+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:44 smithi155 bash[59495]: cluster 2023-10-09T06:37:43.320394+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.424 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:44.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:44.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: cluster 2023-10-09T06:37:43.205881+0000 mgr.y (mgr.25723) 992 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 1021 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: audit 2023-10-09T06:37:43.297865+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: cluster 2023-10-09T06:37:43.297979+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: cluster 2023-10-09T06:37:43.299328+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: cluster 2023-10-09T06:37:43.319216+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[77299]: cluster 2023-10-09T06:37:43.320394+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.588 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: cluster 2023-10-09T06:37:43.205881+0000 mgr.y (mgr.25723) 992 : cluster 0 pgmap v1008: 89 pgs: 89 active+clean; 1021 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-09T06:37:44.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: audit 2023-10-09T06:37:43.297865+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:37:44.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: cluster 2023-10-09T06:37:43.297979+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:37:44.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: cluster 2023-10-09T06:37:43.299328+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: cluster 2023-10-09T06:37:43.319216+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:44 smithi125 bash[80002]: cluster 2023-10-09T06:37:43.320394+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e208 with expected crc 2023-10-09T06:37:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:44.691 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:44.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:44.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:44.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:44.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:44.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:44.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:45.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:45.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.244 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.303943+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:37:45.320 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.308739+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.311271+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.312717+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.312733+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.313494+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.315003+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.316484+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.322 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[77299]: cluster 2023-10-09T06:37:44.317429+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:45.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:45.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:45.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.303943+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:37:45.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.308739+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.311271+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.312717+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.312733+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.313494+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.315003+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.316484+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:45 smithi155 bash[59495]: cluster 2023-10-09T06:37:44.317429+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:45.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:45.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.303943+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:37:45.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.308739+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.311271+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.312717+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.312733+0000 osd.4 (osd.4) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.313494+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.315003+0000 osd.5 (osd.5) 137 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.316484+0000 osd.4 (osd.4) 134 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:45 smithi125 bash[80002]: cluster 2023-10-09T06:37:44.317429+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e209 with expected crc 2023-10-09T06:37:45.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:45.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:45.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:45.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:45.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:46.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:46.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:46.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:46.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:46.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:46.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:46.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:46.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:46 smithi125 bash[77299]: cluster 2023-10-09T06:37:45.206442+0000 mgr.y (mgr.25723) 993 : cluster 0 pgmap v1011: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 900 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:46.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:46 smithi125 bash[80002]: cluster 2023-10-09T06:37:45.206442+0000 mgr.y (mgr.25723) 993 : cluster 0 pgmap v1011: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 900 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:46.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:46.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:46.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:46.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:46.589 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:46 smithi155 bash[59495]: cluster 2023-10-09T06:37:45.206442+0000 mgr.y (mgr.25723) 993 : cluster 0 pgmap v1011: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 900 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:46.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:46.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:46.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:46.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:46.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:46.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:46.987 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:46.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:47.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:47.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:47.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:47.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:47.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:47.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:47.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:47.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:47.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:47.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:47.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:47.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:47.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:47.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:47.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:47.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:47.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:47.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:48.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:48.032 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:48.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:48.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:48.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:48.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:48.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:48.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:48.326 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:48 smithi155 bash[59495]: cluster 2023-10-09T06:37:47.207077+0000 mgr.y (mgr.25723) 994 : cluster 0 pgmap v1012: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 888 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:48.400 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:48.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:48.532 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:48.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:48.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:48 smithi125 bash[77299]: cluster 2023-10-09T06:37:47.207077+0000 mgr.y (mgr.25723) 994 : cluster 0 pgmap v1012: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 888 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:48.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:48 smithi125 bash[80002]: cluster 2023-10-09T06:37:47.207077+0000 mgr.y (mgr.25723) 994 : cluster 0 pgmap v1012: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 888 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:48.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:48.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:48.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:48.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:48.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:48.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:48.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:48.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:49.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:49.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:49.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:49.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:49.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:49.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:49.327 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:49 smithi125 bash[77299]: audit 2023-10-09T06:37:48.595900+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:49.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.328 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:49 smithi155 bash[59495]: audit 2023-10-09T06:37:48.595900+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:49.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:49.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:49.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:49.595 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:49.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:49 smithi125 bash[80002]: audit 2023-10-09T06:37:48.595900+0000 mon.a (mon.0) 1187 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:49.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:49.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.762 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:49.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:49.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:49.992 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:50.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:50.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:50.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:50.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:50.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:50.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:50.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:50.409 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:50 smithi125 bash[77299]: cluster 2023-10-09T06:37:49.207671+0000 mgr.y (mgr.25723) 995 : cluster 0 pgmap v1013: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 837 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:50.409 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:50 smithi125 bash[80002]: cluster 2023-10-09T06:37:49.207671+0000 mgr.y (mgr.25723) 995 : cluster 0 pgmap v1013: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 837 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:50.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:50.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:50.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:50.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:50 smithi155 bash[59495]: cluster 2023-10-09T06:37:49.207671+0000 mgr.y (mgr.25723) 995 : cluster 0 pgmap v1013: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 837 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:50.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:50.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:50.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:50.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:50.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:50.953 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:50.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:50.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:51.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:51.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:51.216 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:37:51] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:37:51.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:51.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:51.254 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:51.254 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:51.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:51.361 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:37:51.364 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:37:51.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:51.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:51 smithi155 bash[59495]: audit 2023-10-09T06:37:50.729371+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:51 smithi155 bash[59495]: audit 2023-10-09T06:37:50.734167+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.125:0/3786841569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:51 smithi155 bash[59495]: audit 2023-10-09T06:37:51.209875+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:51.384 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:51 smithi155 bash[59495]: audit 2023-10-09T06:37:51.310148+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.125:0/679541530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:51.409 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:37:51.438 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:37:51.473 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:37:51.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:51.497 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:37:51.511 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[77299]: audit 2023-10-09T06:37:50.729371+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[77299]: audit 2023-10-09T06:37:50.734167+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.125:0/3786841569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[77299]: audit 2023-10-09T06:37:51.209875+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[77299]: audit 2023-10-09T06:37:51.310148+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.125:0/679541530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[80002]: audit 2023-10-09T06:37:50.729371+0000 mon.a (mon.0) 1188 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[80002]: audit 2023-10-09T06:37:50.734167+0000 mon.b (mon.1) 28 : audit 1 from='client.? 172.21.15.125:0/3786841569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[80002]: audit 2023-10-09T06:37:51.209875+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:51.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:51 smithi125 bash[80002]: audit 2023-10-09T06:37:51.310148+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.125:0/679541530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:51.529 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:37:51.557 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:37:51.587 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:37:51.589 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.589 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.589 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.589 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.590 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:37:51.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:51.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:51.695 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:37:51.695 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:37:51.695 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T06:37:51.695 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:37:51.695 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:37:51.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T06:37:51.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:51.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:51.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:51.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:51.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:51.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:52.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:52.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:52.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:52.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:52.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:52.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:52.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:52.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:52.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.208487+0000 mgr.y (mgr.25723) 996 : cluster 0 pgmap v1014: 99 pgs: 99 active+clean; 650 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.334537+0000 mon.a (mon.0) 1190 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: audit 2023-10-09T06:37:51.349261+0000 mon.a (mon.0) 1191 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: audit 2023-10-09T06:37:51.349539+0000 mon.a (mon.0) 1192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.349749+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:37:52.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.351280+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.351745+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.533948+0000 osd.4 (osd.4) 135 : cluster 0 8.2 scrub starts 2023-10-09T06:37:52.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:52 smithi155 bash[59495]: cluster 2023-10-09T06:37:51.616439+0000 osd.4 (osd.4) 136 : cluster 0 8.2 scrub ok 2023-10-09T06:37:52.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:52.660 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.208487+0000 mgr.y (mgr.25723) 996 : cluster 0 pgmap v1014: 99 pgs: 99 active+clean; 650 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.334537+0000 mon.a (mon.0) 1190 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: audit 2023-10-09T06:37:51.349261+0000 mon.a (mon.0) 1191 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: audit 2023-10-09T06:37:51.349539+0000 mon.a (mon.0) 1192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.349749+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.351280+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.351745+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.533948+0000 osd.4 (osd.4) 135 : cluster 0 8.2 scrub starts 2023-10-09T06:37:52.661 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[80002]: cluster 2023-10-09T06:37:51.616439+0000 osd.4 (osd.4) 136 : cluster 0 8.2 scrub ok 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.208487+0000 mgr.y (mgr.25723) 996 : cluster 0 pgmap v1014: 99 pgs: 99 active+clean; 650 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.334537+0000 mon.a (mon.0) 1190 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: audit 2023-10-09T06:37:51.349261+0000 mon.a (mon.0) 1191 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: audit 2023-10-09T06:37:51.349539+0000 mon.a (mon.0) 1192 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.349749+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.351280+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.351745+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e210 with expected crc 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.533948+0000 osd.4 (osd.4) 135 : cluster 0 8.2 scrub starts 2023-10-09T06:37:52.662 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:52 smithi125 bash[77299]: cluster 2023-10-09T06:37:51.616439+0000 osd.4 (osd.4) 136 : cluster 0 8.2 scrub ok 2023-10-09T06:37:52.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:52.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:52.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:52.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:52.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:52.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:53.051 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:53.077 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:53.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:53.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:53.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:53.344 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:53.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:53.460 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.358704+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:37:53.460 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.366834+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.460 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.370933+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.370934+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.371198+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.373538+0000 osd.3 (osd.3) 1 : cluster 0 8.7 scrub starts 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.379442+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.380895+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.408140+0000 osd.3 (osd.3) 2 : cluster 0 8.7 scrub ok 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.509718+0000 osd.4 (osd.4) 138 : cluster 0 8.12 scrub starts 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.552134+0000 osd.4 (osd.4) 139 : cluster 0 8.12 scrub ok 2023-10-09T06:37:53.461 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:53 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.919143+0000 osd.5 (osd.5) 140 : cluster 0 8.8 scrub starts 2023-10-09T06:37:53.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:53.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:53.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.358704+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:37:53.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.366834+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.370933+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.370934+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.371198+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.373538+0000 osd.3 (osd.3) 1 : cluster 0 8.7 scrub starts 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.379442+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.380895+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.408140+0000 osd.3 (osd.3) 2 : cluster 0 8.7 scrub ok 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.509718+0000 osd.4 (osd.4) 138 : cluster 0 8.12 scrub starts 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.552134+0000 osd.4 (osd.4) 139 : cluster 0 8.12 scrub ok 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.919143+0000 osd.5 (osd.5) 140 : cluster 0 8.8 scrub starts 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.358704+0000 mon.a (mon.0) 1194 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:37:53.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.366834+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.370933+0000 osd.4 (osd.4) 137 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.370934+0000 osd.5 (osd.5) 139 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.371198+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.373538+0000 osd.3 (osd.3) 1 : cluster 0 8.7 scrub starts 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.379442+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.380895+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e211 with expected crc 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.408140+0000 osd.3 (osd.3) 2 : cluster 0 8.7 scrub ok 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.509718+0000 osd.4 (osd.4) 138 : cluster 0 8.12 scrub starts 2023-10-09T06:37:53.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.552134+0000 osd.4 (osd.4) 139 : cluster 0 8.12 scrub ok 2023-10-09T06:37:53.598 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:53 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.919143+0000 osd.5 (osd.5) 140 : cluster 0 8.8 scrub starts 2023-10-09T06:37:53.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:53.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:53.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:53.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:53.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:37:53.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:53.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:53.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:53.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:54.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:54.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:54.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:54.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:54.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:54.552 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:54.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.975272+0000 osd.5 (osd.5) 141 : cluster 0 8.8 scrub ok 2023-10-09T06:37:54.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:52.999036+0000 osd.6 (osd.6) 151 : cluster 0 8.5 scrub starts 2023-10-09T06:37:54.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.037770+0000 osd.6 (osd.6) 152 : cluster 0 8.5 scrub ok 2023-10-09T06:37:54.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.209184+0000 mgr.y (mgr.25723) 997 : cluster 0 pgmap v1017: 99 pgs: 1 remapped+peering, 98 active+clean; 571 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.377111+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.377623+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.378231+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.378947+0000 osd.1 (osd.1) 1 : cluster 0 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.391100+0000 osd.1 (osd.1) 2 : cluster 0 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.475698+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: cluster 2023-10-09T06:37:53.899102+0000 osd.5 (osd.5) 143 : cluster 0 8.4 scrub starts 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[77299]: audit 2023-10-09T06:37:54.121051+0000 mon.a (mon.0) 1196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:54.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.975272+0000 osd.5 (osd.5) 141 : cluster 0 8.8 scrub ok 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:52.999036+0000 osd.6 (osd.6) 151 : cluster 0 8.5 scrub starts 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.037770+0000 osd.6 (osd.6) 152 : cluster 0 8.5 scrub ok 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.209184+0000 mgr.y (mgr.25723) 997 : cluster 0 pgmap v1017: 99 pgs: 1 remapped+peering, 98 active+clean; 571 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.377111+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.377623+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.378231+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.378947+0000 osd.1 (osd.1) 1 : cluster 0 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.391100+0000 osd.1 (osd.1) 2 : cluster 0 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.475698+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: cluster 2023-10-09T06:37:53.899102+0000 osd.5 (osd.5) 143 : cluster 0 8.4 scrub starts 2023-10-09T06:37:54.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:54 smithi125 bash[80002]: audit 2023-10-09T06:37:54.121051+0000 mon.a (mon.0) 1196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.975272+0000 osd.5 (osd.5) 141 : cluster 0 8.8 scrub ok 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:52.999036+0000 osd.6 (osd.6) 151 : cluster 0 8.5 scrub starts 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.037770+0000 osd.6 (osd.6) 152 : cluster 0 8.5 scrub ok 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.209184+0000 mgr.y (mgr.25723) 997 : cluster 0 pgmap v1017: 99 pgs: 1 remapped+peering, 98 active+clean; 571 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.377111+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.377623+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.378231+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.378947+0000 osd.1 (osd.1) 1 : cluster 0 8.10 starting backfill to osd.4 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.391100+0000 osd.1 (osd.1) 2 : cluster 0 8.10 starting backfill to osd.7 from (0'0,0'0] MAX to 209'13076 2023-10-09T06:37:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.475698+0000 osd.4 (osd.4) 140 : cluster 3 failed to encode map e212 with expected crc 2023-10-09T06:37:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: cluster 2023-10-09T06:37:53.899102+0000 osd.5 (osd.5) 143 : cluster 0 8.4 scrub starts 2023-10-09T06:37:54.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:54 smithi155 bash[59495]: audit 2023-10-09T06:37:54.121051+0000 mon.a (mon.0) 1196 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:37:54.655 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:54.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:54.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:54.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:54.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:55.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:55.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:55.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:55.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:55.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:55.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:55.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:55.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:55 smithi125 bash[77299]: cluster 2023-10-09T06:37:54.008299+0000 osd.5 (osd.5) 144 : cluster 0 8.4 scrub ok 2023-10-09T06:37:55.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:55.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:55.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:55.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:55 smithi155 bash[59495]: cluster 2023-10-09T06:37:54.008299+0000 osd.5 (osd.5) 144 : cluster 0 8.4 scrub ok 2023-10-09T06:37:55.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:55.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:55 smithi125 bash[80002]: cluster 2023-10-09T06:37:54.008299+0000 osd.5 (osd.5) 144 : cluster 0 8.4 scrub ok 2023-10-09T06:37:55.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:55.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:55.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:37:55.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:55.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:37:55.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:55.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:56.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:56.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:56.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:56.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:56.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:56.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:56.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:56.482 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:37:56.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:56.591 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:56.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:56 smithi155 bash[59495]: cluster 2023-10-09T06:37:54.997330+0000 osd.6 (osd.6) 154 : cluster 0 8.15 scrub starts 2023-10-09T06:37:56.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:56 smithi155 bash[59495]: cluster 2023-10-09T06:37:55.210012+0000 mgr.y (mgr.25723) 998 : cluster 0 pgmap v1019: 99 pgs: 1 remapped+peering, 98 active+clean; 466 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:56.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:56 smithi155 bash[59495]: cluster 2023-10-09T06:37:55.214931+0000 osd.0 (osd.0) 1 : cluster 0 8.1 scrub starts 2023-10-09T06:37:56.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:56 smithi155 bash[59495]: cluster 2023-10-09T06:37:55.411763+0000 osd.3 (osd.3) 3 : cluster 0 8.0 scrub starts 2023-10-09T06:37:56.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:56.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:56.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:56.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:56.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:56.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:56.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:56.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[77299]: cluster 2023-10-09T06:37:54.997330+0000 osd.6 (osd.6) 154 : cluster 0 8.15 scrub starts 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[77299]: cluster 2023-10-09T06:37:55.210012+0000 mgr.y (mgr.25723) 998 : cluster 0 pgmap v1019: 99 pgs: 1 remapped+peering, 98 active+clean; 466 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[77299]: cluster 2023-10-09T06:37:55.214931+0000 osd.0 (osd.0) 1 : cluster 0 8.1 scrub starts 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[77299]: cluster 2023-10-09T06:37:55.411763+0000 osd.3 (osd.3) 3 : cluster 0 8.0 scrub starts 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[80002]: cluster 2023-10-09T06:37:54.997330+0000 osd.6 (osd.6) 154 : cluster 0 8.15 scrub starts 2023-10-09T06:37:56.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[80002]: cluster 2023-10-09T06:37:55.210012+0000 mgr.y (mgr.25723) 998 : cluster 0 pgmap v1019: 99 pgs: 1 remapped+peering, 98 active+clean; 466 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:37:56.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[80002]: cluster 2023-10-09T06:37:55.214931+0000 osd.0 (osd.0) 1 : cluster 0 8.1 scrub starts 2023-10-09T06:37:56.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:56 smithi125 bash[80002]: cluster 2023-10-09T06:37:55.411763+0000 osd.3 (osd.3) 3 : cluster 0 8.0 scrub starts 2023-10-09T06:37:56.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:57.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:57.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:57.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:57.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:57.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:57.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:57.404 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:57.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:57.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:57.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.027188+0000 osd.6 (osd.6) 155 : cluster 0 8.15 scrub starts 2023-10-09T06:37:57.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.171055+0000 osd.0 (osd.0) 2 : cluster 0 8.6 scrub starts 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.218741+0000 osd.0 (osd.0) 3 : cluster 0 8.6 scrub ok 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.395807+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.398286+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.411253+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.413550+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.413747+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.448782+0000 osd.3 (osd.3) 4 : cluster 0 8.17 scrub starts 2023-10-09T06:37:57.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:56.536960+0000 osd.3 (osd.3) 5 : cluster 0 8.17 scrub ok 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.210742+0000 mgr.y (mgr.25723) 999 : cluster 0 pgmap v1021: 99 pgs: 1 remapped+peering, 98 active+clean; 456 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.027188+0000 osd.6 (osd.6) 155 : cluster 0 8.15 scrub starts 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.171055+0000 osd.0 (osd.0) 2 : cluster 0 8.6 scrub starts 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.218741+0000 osd.0 (osd.0) 3 : cluster 0 8.6 scrub ok 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.395807+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:37:57.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.398286+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.411253+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.413550+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.413747+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.448782+0000 osd.3 (osd.3) 4 : cluster 0 8.17 scrub starts 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:56.536960+0000 osd.3 (osd.3) 5 : cluster 0 8.17 scrub ok 2023-10-09T06:37:57.644 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:57 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.210742+0000 mgr.y (mgr.25723) 999 : cluster 0 pgmap v1021: 99 pgs: 1 remapped+peering, 98 active+clean; 456 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:57.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:57.673 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:57.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:57.817 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.027188+0000 osd.6 (osd.6) 155 : cluster 0 8.15 scrub starts 2023-10-09T06:37:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.171055+0000 osd.0 (osd.0) 2 : cluster 0 8.6 scrub starts 2023-10-09T06:37:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.218741+0000 osd.0 (osd.0) 3 : cluster 0 8.6 scrub ok 2023-10-09T06:37:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.395807+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.398286+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.411253+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.413550+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.413747+0000 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.448782+0000 osd.3 (osd.3) 4 : cluster 0 8.17 scrub starts 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:56.536960+0000 osd.3 (osd.3) 5 : cluster 0 8.17 scrub ok 2023-10-09T06:37:57.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:57 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.210742+0000 mgr.y (mgr.25723) 999 : cluster 0 pgmap v1021: 99 pgs: 1 remapped+peering, 98 active+clean; 456 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:57.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:57.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:57.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:58.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:58.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:58.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:58.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:58.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.075543+0000 osd.6 (osd.6) 157 : cluster 0 8.15 scrub starts 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.105724+0000 osd.6 (osd.6) 158 : cluster 0 8.15 scrub ok 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.405658+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.408339+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.410292+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.529 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:58 smithi155 bash[59495]: cluster 2023-10-09T06:37:57.412209+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:58.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.075543+0000 osd.6 (osd.6) 157 : cluster 0 8.15 scrub starts 2023-10-09T06:37:58.593 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.105724+0000 osd.6 (osd.6) 158 : cluster 0 8.15 scrub ok 2023-10-09T06:37:58.593 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.405658+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:37:58.593 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.408339+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.593 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.410292+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.593 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[77299]: cluster 2023-10-09T06:37:57.412209+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.594 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.075543+0000 osd.6 (osd.6) 157 : cluster 0 8.15 scrub starts 2023-10-09T06:37:58.594 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.105724+0000 osd.6 (osd.6) 158 : cluster 0 8.15 scrub ok 2023-10-09T06:37:58.594 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.405658+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:37:58.594 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.408339+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.594 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.410292+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:58 smithi125 bash[80002]: cluster 2023-10-09T06:37:57.412209+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e214 with expected crc 2023-10-09T06:37:58.612 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:58.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:58.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:58.773 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:58.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:58.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:58.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:58.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:58.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:59.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:59.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:59.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:59.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:59.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:59.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:59.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:59.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:59.562 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:59.668 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:59.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:59 smithi155 bash[59495]: cluster 2023-10-09T06:37:58.416147+0000 osd.3 (osd.3) 6 : cluster 0 8.0 scrub starts 2023-10-09T06:37:59.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:59 smithi155 bash[59495]: cluster 2023-10-09T06:37:58.483421+0000 osd.3 (osd.3) 7 : cluster 0 8.0 scrub ok 2023-10-09T06:37:59.678 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:37:59 smithi155 bash[59495]: cluster 2023-10-09T06:37:59.211396+0000 mgr.y (mgr.25723) 1000 : cluster 0 pgmap v1023: 99 pgs: 1 remapped+peering, 98 active+clean; 422 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:59.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:59.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:59.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:59.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:59.839 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:59.839 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[77299]: cluster 2023-10-09T06:37:58.416147+0000 osd.3 (osd.3) 6 : cluster 0 8.0 scrub starts 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[77299]: cluster 2023-10-09T06:37:58.483421+0000 osd.3 (osd.3) 7 : cluster 0 8.0 scrub ok 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[77299]: cluster 2023-10-09T06:37:59.211396+0000 mgr.y (mgr.25723) 1000 : cluster 0 pgmap v1023: 99 pgs: 1 remapped+peering, 98 active+clean; 422 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[80002]: cluster 2023-10-09T06:37:58.416147+0000 osd.3 (osd.3) 6 : cluster 0 8.0 scrub starts 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[80002]: cluster 2023-10-09T06:37:58.483421+0000 osd.3 (osd.3) 7 : cluster 0 8.0 scrub ok 2023-10-09T06:37:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:37:59 smithi125 bash[80002]: cluster 2023-10-09T06:37:59.211396+0000 mgr.y (mgr.25723) 1000 : cluster 0 pgmap v1023: 99 pgs: 1 remapped+peering, 98 active+clean; 422 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-09T06:37:59.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:59.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:37:59.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:00.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:00.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:00.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:00.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:00.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:00.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:00.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:00.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:00.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:00.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:00.746 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:00.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:00.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:00.877 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:38:00.877 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "" 2023-10-09T06:38:00.877 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:38:00.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:00.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:00.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:01.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:01.022 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:00 smithi125 bash[77299]: cluster 2023-10-09T06:37:59.205714+0000 osd.0 (osd.0) 4 : cluster 0 8.11 scrub starts 2023-10-09T06:38:01.023 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:00 smithi125 bash[77299]: cluster 2023-10-09T06:37:59.299702+0000 osd.0 (osd.0) 5 : cluster 0 8.11 scrub ok 2023-10-09T06:38:01.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:00 smithi125 bash[80002]: cluster 2023-10-09T06:37:59.205714+0000 osd.0 (osd.0) 4 : cluster 0 8.11 scrub starts 2023-10-09T06:38:01.023 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:00 smithi125 bash[80002]: cluster 2023-10-09T06:37:59.299702+0000 osd.0 (osd.0) 5 : cluster 0 8.11 scrub ok 2023-10-09T06:38:01.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:01.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:01.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:01.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:00 smithi155 bash[59495]: cluster 2023-10-09T06:37:59.205714+0000 osd.0 (osd.0) 4 : cluster 0 8.11 scrub starts 2023-10-09T06:38:01.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:00 smithi155 bash[59495]: cluster 2023-10-09T06:37:59.299702+0000 osd.0 (osd.0) 5 : cluster 0 8.11 scrub ok 2023-10-09T06:38:01.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:01.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:01.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:01.391 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:01] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:38:01.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:01.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:01.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:01.567 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:01.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:01.693 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:01.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:01.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:01.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:01.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:01.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: cluster 2023-10-09T06:38:00.899944+0000 osd.7 (osd.7) 122 : cluster 0 8.10 scrub starts 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: cluster 2023-10-09T06:38:00.927719+0000 osd.7 (osd.7) 123 : cluster 0 8.10 scrub ok 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.212095+0000 mgr.y (mgr.25723) 1001 : cluster 0 pgmap v1024: 99 pgs: 99 active+clean; 241 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: audit 2023-10-09T06:38:01.213239+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.728713+0000 osd.5 (osd.5) 146 : cluster 0 8.9 scrub starts 2023-10-09T06:38:01.999 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.902867+0000 osd.5 (osd.5) 147 : cluster 0 8.9 scrub ok 2023-10-09T06:38:02.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: cluster 2023-10-09T06:38:00.899944+0000 osd.7 (osd.7) 122 : cluster 0 8.10 scrub starts 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: cluster 2023-10-09T06:38:00.927719+0000 osd.7 (osd.7) 123 : cluster 0 8.10 scrub ok 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.212095+0000 mgr.y (mgr.25723) 1001 : cluster 0 pgmap v1024: 99 pgs: 99 active+clean; 241 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: audit 2023-10-09T06:38:01.213239+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.728713+0000 osd.5 (osd.5) 146 : cluster 0 8.9 scrub starts 2023-10-09T06:38:02.059 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:01 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.902867+0000 osd.5 (osd.5) 147 : cluster 0 8.9 scrub ok 2023-10-09T06:38:02.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:02.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:02.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: cluster 2023-10-09T06:38:00.899944+0000 osd.7 (osd.7) 122 : cluster 0 8.10 scrub starts 2023-10-09T06:38:02.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: cluster 2023-10-09T06:38:00.927719+0000 osd.7 (osd.7) 123 : cluster 0 8.10 scrub ok 2023-10-09T06:38:02.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.212095+0000 mgr.y (mgr.25723) 1001 : cluster 0 pgmap v1024: 99 pgs: 99 active+clean; 241 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:02.198 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: audit 2023-10-09T06:38:01.213239+0000 mon.a (mon.0) 1199 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:38:02.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.728713+0000 osd.5 (osd.5) 146 : cluster 0 8.9 scrub starts 2023-10-09T06:38:02.199 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:01 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.902867+0000 osd.5 (osd.5) 147 : cluster 0 8.9 scrub ok 2023-10-09T06:38:02.225 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:02.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:02.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:02.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:02.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:02.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:02.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:02.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:02.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:02.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:02.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:02.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:03.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:03.091 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.206827+0000 osd.0 (osd.0) 6 : cluster 0 8.16 scrub starts 2023-10-09T06:38:03.091 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.303069+0000 osd.0 (osd.0) 7 : cluster 0 8.16 scrub ok 2023-10-09T06:38:03.091 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: audit 2023-10-09T06:38:01.996438+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:38:03.091 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.996529+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:38:03.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:01.998856+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:02.008181+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:02.012109+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:02.013753+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.092 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:03 smithi155 bash[59495]: cluster 2023-10-09T06:38:02.015320+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:03.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:03.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:03.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:03.335 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.206827+0000 osd.0 (osd.0) 6 : cluster 0 8.16 scrub starts 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.303069+0000 osd.0 (osd.0) 7 : cluster 0 8.16 scrub ok 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: audit 2023-10-09T06:38:01.996438+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.996529+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:01.998856+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:02.008181+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:02.012109+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:02.013753+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[77299]: cluster 2023-10-09T06:38:02.015320+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.206827+0000 osd.0 (osd.0) 6 : cluster 0 8.16 scrub starts 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.303069+0000 osd.0 (osd.0) 7 : cluster 0 8.16 scrub ok 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: audit 2023-10-09T06:38:01.996438+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.996529+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:01.998856+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:02.008181+0000 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:02.012109+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:02.013753+0000 osd.4 (osd.4) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:03 smithi125 bash[80002]: cluster 2023-10-09T06:38:02.015320+0000 osd.4 (osd.4) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-09T06:38:03.348 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:03.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:03.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:03.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:03.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:03.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:03.641 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:03.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:03.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:03.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:03.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:03.911 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:03.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.159 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:02.999387+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.000718+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.003354+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.003522+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.003527+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.005101+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: cluster 2023-10-09T06:38:03.212767+0000 mgr.y (mgr.25723) 1002 : cluster 0 pgmap v1027: 99 pgs: 99 active+clean; 162 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: audit 2023-10-09T06:38:03.213773+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: audit 2023-10-09T06:38:03.596940+0000 mon.a (mon.0) 1204 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:04.160 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:04 smithi155 bash[59495]: audit 2023-10-09T06:38:04.000957+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:38:04.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:04.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:04.343 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:02.999387+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:38:04.344 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.000718+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.344 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.003354+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.344 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.003522+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.344 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.003527+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.344 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.005101+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.345 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: cluster 2023-10-09T06:38:03.212767+0000 mgr.y (mgr.25723) 1002 : cluster 0 pgmap v1027: 99 pgs: 99 active+clean; 162 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:38:04.345 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: audit 2023-10-09T06:38:03.213773+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:38:04.345 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: audit 2023-10-09T06:38:03.596940+0000 mon.a (mon.0) 1204 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:04.345 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[77299]: audit 2023-10-09T06:38:04.000957+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:02.999387+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.000718+0000 osd.4 (osd.4) 143 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.003354+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.003522+0000 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.003527+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.005101+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e216 with expected crc 2023-10-09T06:38:04.346 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: cluster 2023-10-09T06:38:03.212767+0000 mgr.y (mgr.25723) 1002 : cluster 0 pgmap v1027: 99 pgs: 99 active+clean; 162 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:38:04.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: audit 2023-10-09T06:38:03.213773+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:38:04.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: audit 2023-10-09T06:38:03.596940+0000 mon.a (mon.0) 1204 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:04.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:04 smithi125 bash[80002]: audit 2023-10-09T06:38:04.000957+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:38:04.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:04.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:04.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:04.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:04.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:04.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:04.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:05.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:05.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.001039+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.003907+0000 osd.0 (osd.0) 8 : cluster 0 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.007422+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.009939+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.016251+0000 osd.0 (osd.0) 9 : cluster 0 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.026273+0000 osd.0 (osd.0) 10 : cluster 0 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.775313+0000 osd.5 (osd.5) 152 : cluster 0 8.3 scrub starts 2023-10-09T06:38:05.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: cluster 2023-10-09T06:38:04.785870+0000 osd.5 (osd.5) 153 : cluster 0 8.3 scrub ok 2023-10-09T06:38:05.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:05 smithi155 bash[59495]: audit 2023-10-09T06:38:04.981943+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:05.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:05.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:05.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.001039+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.003907+0000 osd.0 (osd.0) 8 : cluster 0 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.007422+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.009939+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.016251+0000 osd.0 (osd.0) 9 : cluster 0 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.026273+0000 osd.0 (osd.0) 10 : cluster 0 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.775313+0000 osd.5 (osd.5) 152 : cluster 0 8.3 scrub starts 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: cluster 2023-10-09T06:38:04.785870+0000 osd.5 (osd.5) 153 : cluster 0 8.3 scrub ok 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[77299]: audit 2023-10-09T06:38:04.981943+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.001039+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.003907+0000 osd.0 (osd.0) 8 : cluster 0 8.11 starting backfill to osd.5 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.007422+0000 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.009939+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e217 with expected crc 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.016251+0000 osd.0 (osd.0) 9 : cluster 0 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.026273+0000 osd.0 (osd.0) 10 : cluster 0 8.11 starting backfill to osd.7 from (0'0,0'0] MAX to 214'13192 2023-10-09T06:38:05.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.775313+0000 osd.5 (osd.5) 152 : cluster 0 8.3 scrub starts 2023-10-09T06:38:05.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: cluster 2023-10-09T06:38:04.785870+0000 osd.5 (osd.5) 153 : cluster 0 8.3 scrub ok 2023-10-09T06:38:05.393 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:05 smithi125 bash[80002]: audit 2023-10-09T06:38:04.981943+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:05.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.475 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:05.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:05.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:05.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:05.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:05.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:05.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:05.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:05.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:06.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:06.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:06.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:06.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:06.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:06.239 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:06.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:06.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.026220+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:38:06.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.032204+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.034957+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.296 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.035810+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.035990+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.036396+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.039119+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: cluster 2023-10-09T06:38:05.213449+0000 mgr.y (mgr.25723) 1003 : cluster 0 pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1449 objects misplaced (1.449%) 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[77299]: audit 2023-10-09T06:38:05.214790+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.026220+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.032204+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.297 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.034957+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.035810+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.035990+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.036396+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.039119+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: cluster 2023-10-09T06:38:05.213449+0000 mgr.y (mgr.25723) 1003 : cluster 0 pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1449 objects misplaced (1.449%) 2023-10-09T06:38:06.298 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:06 smithi125 bash[80002]: audit 2023-10-09T06:38:05.214790+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:38:06.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.026220+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:38:06.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.032204+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.034957+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.035810+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.035990+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.036396+0000 osd.4 (osd.4) 144 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.039119+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e218 with expected crc 2023-10-09T06:38:06.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: cluster 2023-10-09T06:38:05.213449+0000 mgr.y (mgr.25723) 1003 : cluster 0 pgmap v1030: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1449 objects misplaced (1.449%) 2023-10-09T06:38:06.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:06 smithi155 bash[59495]: audit 2023-10-09T06:38:05.214790+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:38:06.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:06.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:06.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:06.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:06.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:06.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:06.887 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:06.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:06.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:07.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:07.052 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:07.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:07.059 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:38:07.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:38:07.060 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-09T06:38:07.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T06:38:07.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:07.184 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:07.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:07.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: audit 2023-10-09T06:38:06.035340+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:38:07.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.035433+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:38:07.298 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.037772+0000 osd.0 (osd.0) 11 : cluster 0 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.040647+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.041978+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.043520+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.044809+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.046307+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.046454+0000 osd.0 (osd.0) 12 : cluster 0 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.054252+0000 osd.0 (osd.0) 13 : cluster 0 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.299 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:07 smithi155 bash[59495]: cluster 2023-10-09T06:38:06.069287+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:07.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:07.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:07.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:07.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:07.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: audit 2023-10-09T06:38:06.035340+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:38:07.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.035433+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:38:07.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.037772+0000 osd.0 (osd.0) 11 : cluster 0 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.616 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.040647+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.041978+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.043520+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.044809+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.046307+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.046454+0000 osd.0 (osd.0) 12 : cluster 0 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.054252+0000 osd.0 (osd.0) 13 : cluster 0 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[77299]: cluster 2023-10-09T06:38:06.069287+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: audit 2023-10-09T06:38:06.035340+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.035433+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.037772+0000 osd.0 (osd.0) 11 : cluster 0 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.040647+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.041978+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.043520+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.044809+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.046307+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.046454+0000 osd.0 (osd.0) 12 : cluster 0 8.12 starting backfill to osd.6 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.054252+0000 osd.0 (osd.0) 13 : cluster 0 8.12 starting backfill to osd.7 from (0'0,0'0] MAX to 215'13304 2023-10-09T06:38:07.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:07 smithi125 bash[80002]: cluster 2023-10-09T06:38:06.069287+0000 osd.4 (osd.4) 145 : cluster 3 failed to encode map e219 with expected crc 2023-10-09T06:38:07.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:07.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:07.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:07.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:07.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:07.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:07.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:07.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:08.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:08.118 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:08.142 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:08.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.036349+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.038711+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.039075+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.214135+0000 mgr.y (mgr.25723) 1004 : cluster 0 pgmap v1033: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 27 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1182 objects misplaced (1.777%) 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: audit 2023-10-09T06:38:07.215149+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.833857+0000 osd.5 (osd.5) 161 : cluster 0 8.18 scrub starts 2023-10-09T06:38:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:08 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.855119+0000 osd.5 (osd.5) 162 : cluster 0 8.18 scrub ok 2023-10-09T06:38:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:08.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:08.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:08.369 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.036349+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:38:08.369 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.038711+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.369 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.039075+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.369 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.214135+0000 mgr.y (mgr.25723) 1004 : cluster 0 pgmap v1033: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 27 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1182 objects misplaced (1.777%) 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: audit 2023-10-09T06:38:07.215149+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.833857+0000 osd.5 (osd.5) 161 : cluster 0 8.18 scrub starts 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.855119+0000 osd.5 (osd.5) 162 : cluster 0 8.18 scrub ok 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.036349+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.038711+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.370 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.039075+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e220 with expected crc 2023-10-09T06:38:08.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.214135+0000 mgr.y (mgr.25723) 1004 : cluster 0 pgmap v1033: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 27 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 21/1182 objects misplaced (1.777%) 2023-10-09T06:38:08.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: audit 2023-10-09T06:38:07.215149+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:38:08.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.833857+0000 osd.5 (osd.5) 161 : cluster 0 8.18 scrub starts 2023-10-09T06:38:08.371 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:08 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.855119+0000 osd.5 (osd.5) 162 : cluster 0 8.18 scrub ok 2023-10-09T06:38:08.379 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:08.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:08.452 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 109065 objects 2023-10-09T06:38:08.507 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :161.635 2023-10-09T06:38:08.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:08.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:08.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:08.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:08.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:08.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:08.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:08.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:08.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:08.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:09.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:09.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:09.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:09.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:09.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:09.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.244229+0000 osd.0 (osd.0) 14 : cluster 0 8.1 scrub starts 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:07.259246+0000 osd.0 (osd.0) 15 : cluster 0 8.1 scrub ok 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: audit 2023-10-09T06:38:08.050914+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.051003+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.053140+0000 osd.1 (osd.1) 3 : cluster 0 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.053325+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.345 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.054518+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.056733+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.057530+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.064304+0000 osd.1 (osd.1) 4 : cluster 0 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:09 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.977143+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-09T06:38:09.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:09.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.244229+0000 osd.0 (osd.0) 14 : cluster 0 8.1 scrub starts 2023-10-09T06:38:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:07.259246+0000 osd.0 (osd.0) 15 : cluster 0 8.1 scrub ok 2023-10-09T06:38:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: audit 2023-10-09T06:38:08.050914+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:38:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.051003+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:38:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.053140+0000 osd.1 (osd.1) 3 : cluster 0 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.392 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.053325+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.054518+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.056733+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.057530+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.393 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.064304+0000 osd.1 (osd.1) 4 : cluster 0 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.394 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.977143+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-09T06:38:09.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.244229+0000 osd.0 (osd.0) 14 : cluster 0 8.1 scrub starts 2023-10-09T06:38:09.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:07.259246+0000 osd.0 (osd.0) 15 : cluster 0 8.1 scrub ok 2023-10-09T06:38:09.394 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: audit 2023-10-09T06:38:08.050914+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:38:09.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.051003+0000 mon.a (mon.0) 1215 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:38:09.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.053140+0000 osd.1 (osd.1) 3 : cluster 0 8.13 starting backfill to osd.2 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.053325+0000 osd.4 (osd.4) 146 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.395 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.054518+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.056733+0000 osd.4 (osd.4) 147 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.057530+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e221 with expected crc 2023-10-09T06:38:09.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.064304+0000 osd.1 (osd.1) 4 : cluster 0 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 217'13057 2023-10-09T06:38:09.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:09 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.977143+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-09T06:38:09.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:09.503 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:09.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:09.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:09.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:09.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:09.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:09.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:09.897 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:38:09.897 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:38:09.897 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T06:38:09.897 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:38:09.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:09.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:10.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:10.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:10.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.168 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:10.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.979635+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.980217+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.983655+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:08.991897+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: audit 2023-10-09T06:38:09.138584+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:10.283 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.214881+0000 mgr.y (mgr.25723) 1005 : cluster 0 pgmap v1036: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 17 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 21/696 objects misplaced (3.017%) 2023-10-09T06:38:10.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.796008+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.284 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:10 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.986093+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-09T06:38:10.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:10.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.440 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.979635+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.980217+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.983655+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:08.991897+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: audit 2023-10-09T06:38:09.138584+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.214881+0000 mgr.y (mgr.25723) 1005 : cluster 0 pgmap v1036: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 17 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 21/696 objects misplaced (3.017%) 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.796008+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.986093+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.979635+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.441 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.980217+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.983655+0000 osd.4 (osd.4) 148 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:08.991897+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: audit 2023-10-09T06:38:09.138584+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.214881+0000 mgr.y (mgr.25723) 1005 : cluster 0 pgmap v1036: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 17 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 21/696 objects misplaced (3.017%) 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.796008+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e222 with expected crc 2023-10-09T06:38:10.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:10 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.986093+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-09T06:38:10.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:10.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:10.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:10.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:10.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:10.939 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:10.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:10.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:10.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:11.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:11.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:11.212 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.988257+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.212 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.989062+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.212 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.993643+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.212 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:09.998062+0000 osd.4 (osd.4) 150 : cluster 0 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13207 2023-10-09T06:38:11.213 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:10.234120+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.213 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:10.406070+0000 osd.4 (osd.4) 151 : cluster 0 8.14 scrub starts 2023-10-09T06:38:11.213 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:10.425943+0000 osd.4 (osd.4) 152 : cluster 0 8.14 scrub ok 2023-10-09T06:38:11.213 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:10.471390+0000 osd.3 (osd.3) 8 : cluster 0 8.13 scrub starts 2023-10-09T06:38:11.213 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:11 smithi155 bash[59495]: cluster 2023-10-09T06:38:10.489805+0000 osd.3 (osd.3) 9 : cluster 0 8.13 scrub ok 2023-10-09T06:38:11.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:11.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:11.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:11.350 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:11] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-10-09T06:38:11.351 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.988257+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.351 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.989062+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.351 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.993643+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.351 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:09.998062+0000 osd.4 (osd.4) 150 : cluster 0 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13207 2023-10-09T06:38:11.351 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:10.234120+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:10.406070+0000 osd.4 (osd.4) 151 : cluster 0 8.14 scrub starts 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:10.425943+0000 osd.4 (osd.4) 152 : cluster 0 8.14 scrub ok 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:10.471390+0000 osd.3 (osd.3) 8 : cluster 0 8.13 scrub starts 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[77299]: cluster 2023-10-09T06:38:10.489805+0000 osd.3 (osd.3) 9 : cluster 0 8.13 scrub ok 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.988257+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.989062+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.993643+0000 osd.4 (osd.4) 149 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:09.998062+0000 osd.4 (osd.4) 150 : cluster 0 8.14 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13207 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:10.234120+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e223 with expected crc 2023-10-09T06:38:11.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:10.406070+0000 osd.4 (osd.4) 151 : cluster 0 8.14 scrub starts 2023-10-09T06:38:11.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:10.425943+0000 osd.4 (osd.4) 152 : cluster 0 8.14 scrub ok 2023-10-09T06:38:11.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:10.471390+0000 osd.3 (osd.3) 8 : cluster 0 8.13 scrub starts 2023-10-09T06:38:11.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:11 smithi125 bash[80002]: cluster 2023-10-09T06:38:10.489805+0000 osd.3 (osd.3) 9 : cluster 0 8.13 scrub ok 2023-10-09T06:38:11.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:11.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:11.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:11.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:11.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:11.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:11.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:11.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:11.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:12.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:12.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:12.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:12.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:12.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:12.276 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.156357+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-09T06:38:12.276 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.162403+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.165758+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.215387+0000 mgr.y (mgr.25723) 1006 : cluster 0 pgmap v1039: 99 pgs: 2 remapped+peering, 97 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: audit 2023-10-09T06:38:11.222817+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: audit 2023-10-09T06:38:11.231334+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.830747+0000 osd.5 (osd.5) 166 : cluster 0 8.19 scrub starts 2023-10-09T06:38:12.277 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:12 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.851705+0000 osd.5 (osd.5) 167 : cluster 0 8.19 scrub ok 2023-10-09T06:38:12.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:12.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:12.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:12.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:12.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:12.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.156357+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-09T06:38:12.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.162403+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.165758+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.215387+0000 mgr.y (mgr.25723) 1006 : cluster 0 pgmap v1039: 99 pgs: 2 remapped+peering, 97 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: audit 2023-10-09T06:38:11.222817+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: audit 2023-10-09T06:38:11.231334+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.830747+0000 osd.5 (osd.5) 166 : cluster 0 8.19 scrub starts 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.851705+0000 osd.5 (osd.5) 167 : cluster 0 8.19 scrub ok 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.156357+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.162403+0000 osd.4 (osd.4) 153 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.165758+0000 osd.4 (osd.4) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:12.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.215387+0000 mgr.y (mgr.25723) 1006 : cluster 0 pgmap v1039: 99 pgs: 2 remapped+peering, 97 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2023-10-09T06:38:12.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: audit 2023-10-09T06:38:11.222817+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: audit 2023-10-09T06:38:11.231334+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:12.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.830747+0000 osd.5 (osd.5) 166 : cluster 0 8.19 scrub starts 2023-10-09T06:38:12.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:12 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.851705+0000 osd.5 (osd.5) 167 : cluster 0 8.19 scrub ok 2023-10-09T06:38:12.554 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:12.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:12.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:12.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:12.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:12.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:12.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:12.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:12.995 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T06:38:12.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T06:38:13.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:13.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:13.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:38:13.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:11.934614+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:12.172051+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:12.173878+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:12.176434+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:12.179314+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.193 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: cluster 2023-10-09T06:38:12.353819+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: audit 2023-10-09T06:38:12.473479+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: audit 2023-10-09T06:38:12.479470+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.194 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: audit 2023-10-09T06:38:12.899879+0000 mon.a (mon.0) 1225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:13.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:13.291 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:38:13.308 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:13.320 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:38:13.330 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:13.348 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:38:13.372 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:38:13.412 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:38:13.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:11.934614+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:12.172051+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:12.173878+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:12.176434+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:12.179314+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: cluster 2023-10-09T06:38:12.353819+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: audit 2023-10-09T06:38:12.473479+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: audit 2023-10-09T06:38:12.479470+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: audit 2023-10-09T06:38:12.899879+0000 mon.a (mon.0) 1225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:13.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: audit 2023-10-09T06:38:12.900718+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:13.448 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:13 smithi155 bash[59495]: audit 2023-10-09T06:38:12.906178+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.450 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:38:13.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:13.482 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:38:13.508 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.510 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:38:13.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:11.934614+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e224 with expected crc 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:12.172051+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:12.173878+0000 osd.4 (osd.4) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:12.176434+0000 osd.4 (osd.4) 156 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:12.179314+0000 osd.4 (osd.4) 157 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: cluster 2023-10-09T06:38:12.353819+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e225 with expected crc 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: audit 2023-10-09T06:38:12.473479+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: audit 2023-10-09T06:38:12.479470+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: audit 2023-10-09T06:38:12.899879+0000 mon.a (mon.0) 1225 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: audit 2023-10-09T06:38:12.900718+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:13.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[77299]: audit 2023-10-09T06:38:12.906178+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: audit 2023-10-09T06:38:12.900718+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:13.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:13 smithi125 bash[80002]: audit 2023-10-09T06:38:12.906178+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:13.564 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:13.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:13.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:13.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:13.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:13.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:13.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:13.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:13.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:13.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:14.000 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:14.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:14.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:14.194 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:38:14.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:14 smithi155 bash[59495]: cluster 2023-10-09T06:38:13.216092+0000 mgr.y (mgr.25723) 1007 : cluster 0 pgmap v1041: 99 pgs: 1 remapped+peering, 98 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s, 1 objects/s recovering 2023-10-09T06:38:14.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:14 smithi155 bash[59495]: audit 2023-10-09T06:38:13.257777+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.125:0/4076407271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:14.251 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:14 smithi155 bash[59495]: audit 2023-10-09T06:38:13.703114+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.125:0/3068637922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.252 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:14 smithi155 bash[59495]: audit 2023-10-09T06:38:13.703477+0000 mon.a (mon.0) 1228 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:14.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:14.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[77299]: cluster 2023-10-09T06:38:13.216092+0000 mgr.y (mgr.25723) 1007 : cluster 0 pgmap v1041: 99 pgs: 1 remapped+peering, 98 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s, 1 objects/s recovering 2023-10-09T06:38:14.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[77299]: audit 2023-10-09T06:38:13.257777+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.125:0/4076407271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:14.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[77299]: audit 2023-10-09T06:38:13.703114+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.125:0/3068637922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[77299]: audit 2023-10-09T06:38:13.703477+0000 mon.a (mon.0) 1228 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[80002]: cluster 2023-10-09T06:38:13.216092+0000 mgr.y (mgr.25723) 1007 : cluster 0 pgmap v1041: 99 pgs: 1 remapped+peering, 98 active+clean; 5.4 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 121 KiB/s, 1 objects/s recovering 2023-10-09T06:38:14.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[80002]: audit 2023-10-09T06:38:13.257777+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.125:0/4076407271' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:14.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[80002]: audit 2023-10-09T06:38:13.703114+0000 mon.c (mon.2) 95 : audit 1 from='client.? 172.21.15.125:0/3068637922' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:14 smithi125 bash[80002]: audit 2023-10-09T06:38:13.703477+0000 mon.a (mon.0) 1228 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:14.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:14.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:14.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:14.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:14.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.683 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:14.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:14.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:14.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:14.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:15.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:15.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:15.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:15.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:15.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:15.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[77299]: cluster 2023-10-09T06:38:14.168692+0000 mon.a (mon.0) 1229 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:15.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[77299]: cluster 2023-10-09T06:38:14.168724+0000 mon.a (mon.0) 1230 : cluster 1 Cluster is now healthy 2023-10-09T06:38:15.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[77299]: cluster 2023-10-09T06:38:14.177436+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:15.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[77299]: audit 2023-10-09T06:38:14.178297+0000 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:15.324 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[77299]: cluster 2023-10-09T06:38:14.178456+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-09T06:38:15.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[80002]: cluster 2023-10-09T06:38:14.168692+0000 mon.a (mon.0) 1229 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:15.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[80002]: cluster 2023-10-09T06:38:14.168724+0000 mon.a (mon.0) 1230 : cluster 1 Cluster is now healthy 2023-10-09T06:38:15.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[80002]: cluster 2023-10-09T06:38:14.177436+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:15.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[80002]: audit 2023-10-09T06:38:14.178297+0000 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:15.325 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:15 smithi125 bash[80002]: cluster 2023-10-09T06:38:14.178456+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-09T06:38:15.339 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:15.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:15.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:15 smithi155 bash[59495]: cluster 2023-10-09T06:38:14.168692+0000 mon.a (mon.0) 1229 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:15 smithi155 bash[59495]: cluster 2023-10-09T06:38:14.168724+0000 mon.a (mon.0) 1230 : cluster 1 Cluster is now healthy 2023-10-09T06:38:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:15 smithi155 bash[59495]: cluster 2023-10-09T06:38:14.177436+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:15 smithi155 bash[59495]: audit 2023-10-09T06:38:14.178297+0000 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:15.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:15 smithi155 bash[59495]: cluster 2023-10-09T06:38:14.178456+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-09T06:38:15.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:15.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:15.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:15.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:15.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:15.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:15.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:15.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:16.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:16.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:16.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[77299]: cluster 2023-10-09T06:38:15.105346+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:16.246 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[77299]: cluster 2023-10-09T06:38:15.217266+0000 mgr.y (mgr.25723) 1008 : cluster 0 pgmap v1043: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:38:16.247 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[77299]: audit 2023-10-09T06:38:15.218897+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:38:16.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[80002]: cluster 2023-10-09T06:38:15.105346+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:16.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[80002]: cluster 2023-10-09T06:38:15.217266+0000 mgr.y (mgr.25723) 1008 : cluster 0 pgmap v1043: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:38:16.247 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:16 smithi125 bash[80002]: audit 2023-10-09T06:38:15.218897+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:38:16.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:16.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:16.385 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:16.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:16 smithi155 bash[59495]: cluster 2023-10-09T06:38:15.105346+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e226 with expected crc 2023-10-09T06:38:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:16 smithi155 bash[59495]: cluster 2023-10-09T06:38:15.217266+0000 mgr.y (mgr.25723) 1008 : cluster 0 pgmap v1043: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:38:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:16 smithi155 bash[59495]: audit 2023-10-09T06:38:15.218897+0000 mon.a (mon.0) 1233 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:38:16.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:16.521 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:16.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:16.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:16.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:16.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:16.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:16.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:17.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:17.060 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:17.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:17.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:17.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[77299]: audit 2023-10-09T06:38:16.201801+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:38:17.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[77299]: cluster 2023-10-09T06:38:16.201875+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-10-09T06:38:17.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[77299]: cluster 2023-10-09T06:38:16.204394+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[77299]: cluster 2023-10-09T06:38:16.208552+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.248 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[77299]: cluster 2023-10-09T06:38:16.209155+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[80002]: audit 2023-10-09T06:38:16.201801+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:38:17.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[80002]: cluster 2023-10-09T06:38:16.201875+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-10-09T06:38:17.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[80002]: cluster 2023-10-09T06:38:16.204394+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[80002]: cluster 2023-10-09T06:38:16.208552+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.249 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:17 smithi125 bash[80002]: cluster 2023-10-09T06:38:16.209155+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:17.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:17.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:17.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:17 smithi155 bash[59495]: audit 2023-10-09T06:38:16.201801+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:38:17.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:17 smithi155 bash[59495]: cluster 2023-10-09T06:38:16.201875+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2023-10-09T06:38:17.450 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:17 smithi155 bash[59495]: cluster 2023-10-09T06:38:16.204394+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.450 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:17 smithi155 bash[59495]: cluster 2023-10-09T06:38:16.208552+0000 osd.4 (osd.4) 158 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.450 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:17 smithi155 bash[59495]: cluster 2023-10-09T06:38:16.209155+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e227 with expected crc 2023-10-09T06:38:17.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:17.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:17.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:17.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:17.733 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:17.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:17.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:17.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:17.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:17.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:38:18.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:18.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:18.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:18.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:18.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:18.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.204972+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-10-09T06:38:18.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.206126+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.209385+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.210640+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.213976+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.213980+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[77299]: cluster 2023-10-09T06:38:17.221096+0000 mgr.y (mgr.25723) 1009 : cluster 0 pgmap v1046: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.204972+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-10-09T06:38:18.306 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.206126+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.209385+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.210640+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.213976+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.213980+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:18 smithi125 bash[80002]: cluster 2023-10-09T06:38:17.221096+0000 mgr.y (mgr.25723) 1009 : cluster 0 pgmap v1046: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T06:38:18.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:18.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:18.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:18.523 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.204972+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2023-10-09T06:38:18.523 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.206126+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.523 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.209385+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.524 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.210640+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.524 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.213976+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.524 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.213980+0000 osd.4 (osd.4) 159 : cluster 3 failed to encode map e228 with expected crc 2023-10-09T06:38:18.524 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:18 smithi155 bash[59495]: cluster 2023-10-09T06:38:17.221096+0000 mgr.y (mgr.25723) 1009 : cluster 0 pgmap v1046: 99 pgs: 99 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T06:38:18.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:18.645 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:18.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:18.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:38:18.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:18.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:18.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:18.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:18.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:19.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:19.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:19.186 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:19.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:19.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:19.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:19.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.221525+0000 osd.3 (osd.3) 10 : cluster 0 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:19.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.223344+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-10-09T06:38:19.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.223872+0000 osd.1 (osd.1) 5 : cluster 0 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.225304+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.226932+0000 osd.0 (osd.0) 16 : cluster 0 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.228940+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.231713+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.233154+0000 osd.7 (osd.7) 135 : cluster 0 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13096 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.234652+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.237459+0000 osd.1 (osd.1) 6 : cluster 0 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.237731+0000 osd.5 (osd.5) 172 : cluster 0 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.237991+0000 osd.0 (osd.0) 17 : cluster 0 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.250445+0000 osd.0 (osd.0) 18 : cluster 0 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: audit 2023-10-09T06:38:18.601508+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: audit 2023-10-09T06:38:18.603403+0000 mon.a (mon.0) 1239 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:19.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.982549+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-10-09T06:38:19.396 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:19 smithi155 bash[59495]: audit 2023-10-09T06:38:19.155739+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:19.486 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:19.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:19.600 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.221525+0000 osd.3 (osd.3) 10 : cluster 0 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.223344+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.223872+0000 osd.1 (osd.1) 5 : cluster 0 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.225304+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.226932+0000 osd.0 (osd.0) 16 : cluster 0 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.228940+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.231713+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.233154+0000 osd.7 (osd.7) 135 : cluster 0 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13096 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.234652+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.237459+0000 osd.1 (osd.1) 6 : cluster 0 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.237731+0000 osd.5 (osd.5) 172 : cluster 0 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:19.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.237991+0000 osd.0 (osd.0) 17 : cluster 0 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.250445+0000 osd.0 (osd.0) 18 : cluster 0 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: audit 2023-10-09T06:38:18.601508+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: audit 2023-10-09T06:38:18.603403+0000 mon.a (mon.0) 1239 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.982549+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[80002]: audit 2023-10-09T06:38:19.155739+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.221525+0000 osd.3 (osd.3) 10 : cluster 0 8.18 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.223344+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.223872+0000 osd.1 (osd.1) 5 : cluster 0 8.17 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.602 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.225304+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.226932+0000 osd.0 (osd.0) 16 : cluster 0 8.16 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.228940+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.231713+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.233154+0000 osd.7 (osd.7) 135 : cluster 0 8.15 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13096 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.234652+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e229 with expected crc 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.237459+0000 osd.1 (osd.1) 6 : cluster 0 8.17 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13108 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.237731+0000 osd.5 (osd.5) 172 : cluster 0 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.237991+0000 osd.0 (osd.0) 17 : cluster 0 8.16 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.250445+0000 osd.0 (osd.0) 18 : cluster 0 8.16 starting backfill to osd.5 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: audit 2023-10-09T06:38:18.601508+0000 mon.a (mon.0) 1238 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: audit 2023-10-09T06:38:18.603403+0000 mon.a (mon.0) 1239 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:19.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.982549+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2023-10-09T06:38:19.604 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:19 smithi125 bash[77299]: audit 2023-10-09T06:38:19.155739+0000 mon.a (mon.0) 1241 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:19.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:19.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:19.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:19.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:19.854 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:19.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:19.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:19.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:20.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:20.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.155 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:20.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:20.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:20.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.404 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.983618+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.986259+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.988321+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.989076+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.993424+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.995247+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:18.997101+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:19.221691+0000 mgr.y (mgr.25723) 1010 : cluster 0 pgmap v1049: 99 pgs: 2 remapped+peering, 97 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-09T06:38:20.405 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[80002]: cluster 2023-10-09T06:38:19.996454+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.983618+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.986259+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.988321+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.989076+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.993424+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.995247+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:18.997101+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:19.221691+0000 mgr.y (mgr.25723) 1010 : cluster 0 pgmap v1049: 99 pgs: 2 remapped+peering, 97 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: cluster 2023-10-09T06:38:19.996454+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-10-09T06:38:20.406 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:20 smithi125 bash[77299]: debug 2023-10-09T06:38:20.378+0000 7ff949fd9700 -1 mon.a@0(leader).osd e231 definitely_dead 0 2023-10-09T06:38:20.421 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:20.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:20.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.983618+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.986259+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.988321+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.989076+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.993424+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.995247+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:18.997101+0000 osd.4 (osd.4) 160 : cluster 3 failed to encode map e230 with expected crc 2023-10-09T06:38:20.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:19.221691+0000 mgr.y (mgr.25723) 1010 : cluster 0 pgmap v1049: 99 pgs: 2 remapped+peering, 97 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-09T06:38:20.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:20 smithi155 bash[59495]: cluster 2023-10-09T06:38:19.996454+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2023-10-09T06:38:20.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:20.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:20.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:20.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:20.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:20.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:20.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:20.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:21.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:21.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:21.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:21.265 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.5. 2023-10-09T06:38:21.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:21.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:19.998832+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:20.002096+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:20.004483+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:20.006321+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:20.010315+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: audit 2023-10-09T06:38:20.378257+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.125:0/4248867412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: audit 2023-10-09T06:38:20.379133+0000 mon.a (mon.0) 1243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.355 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:21 smithi155 bash[59495]: cluster 2023-10-09T06:38:20.379328+0000 mon.a (mon.0) 1244 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:38:21.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:21.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:21.492 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:21] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T06:38:21.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:19.998832+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:20.002096+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:20.004483+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.492 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:20.006321+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:20.010315+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: audit 2023-10-09T06:38:20.378257+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.125:0/4248867412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: audit 2023-10-09T06:38:20.379133+0000 mon.a (mon.0) 1243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[77299]: cluster 2023-10-09T06:38:20.379328+0000 mon.a (mon.0) 1244 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:19.998832+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:20.002096+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:20.004483+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:20.006321+0000 osd.4 (osd.4) 161 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:20.010315+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e231 with expected crc 2023-10-09T06:38:21.493 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: audit 2023-10-09T06:38:20.378257+0000 mon.c (mon.2) 96 : audit 1 from='client.? 172.21.15.125:0/4248867412' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: audit 2023-10-09T06:38:20.379133+0000 mon.a (mon.0) 1243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:38:21.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:21 smithi125 bash[80002]: cluster 2023-10-09T06:38:20.379328+0000 mon.a (mon.0) 1244 : cluster 1 Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:38:21.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:21.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:21.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:21.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:21.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:21.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:22.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:22.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:22.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:22.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:22.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:22.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:22.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:22.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:22.561 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:22.572 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.222391+0000 mgr.y (mgr.25723) 1011 : cluster 0 pgmap v1051: 99 pgs: 4 peering, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.68k op/s 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: audit 2023-10-09T06:38:21.244341+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.244468+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.251029+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.251100+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.251940+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.253845+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.254082+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.254597+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.573 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.256437+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.258788+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.258837+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.261553+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.261892+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:22 smithi155 bash[59495]: cluster 2023-10-09T06:38:21.263882+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:22.598 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.222391+0000 mgr.y (mgr.25723) 1011 : cluster 0 pgmap v1051: 99 pgs: 4 peering, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.68k op/s 2023-10-09T06:38:22.598 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: audit 2023-10-09T06:38:21.244341+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:38:22.598 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.244468+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.251029+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.251100+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.251940+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.253845+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.254082+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.254597+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.256437+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.258788+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.258837+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.261553+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.261892+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[77299]: cluster 2023-10-09T06:38:21.263882+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.222391+0000 mgr.y (mgr.25723) 1011 : cluster 0 pgmap v1051: 99 pgs: 4 peering, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s wr, 1.68k op/s 2023-10-09T06:38:22.600 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: audit 2023-10-09T06:38:21.244341+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.244468+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.251029+0000 osd.4 (osd.4) 162 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.251100+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.251940+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.253845+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.254082+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.254597+0000 osd.4 (osd.4) 163 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.256437+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.258788+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.258837+0000 osd.4 (osd.4) 164 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.261553+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.261892+0000 osd.4 (osd.4) 165 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.602 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:22 smithi125 bash[80002]: cluster 2023-10-09T06:38:21.263882+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e232 with expected crc 2023-10-09T06:38:22.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:22.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:22.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:22.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:22.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:22.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:22.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:23.016 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:23.016 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:23.030 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:23.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:23.130 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:23.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:23.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:23.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:23.377 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.253376+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.256709+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.256732+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.258904+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.259336+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.259433+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:23 smithi155 bash[59495]: cluster 2023-10-09T06:38:22.268608+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:23.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:23.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.253376+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:38:23.509 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.256709+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.256732+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.258904+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.259336+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.259433+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[77299]: cluster 2023-10-09T06:38:22.268608+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.253376+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.256709+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.256732+0000 osd.4 (osd.4) 166 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.258904+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.259336+0000 osd.4 (osd.4) 167 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.259433+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:23 smithi125 bash[80002]: cluster 2023-10-09T06:38:22.268608+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e233 with expected crc 2023-10-09T06:38:23.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:23.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:23.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:23.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:23.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:23.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:23.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:23.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:24.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:24.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:24.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.186 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:24.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:24.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.223187+0000 mgr.y (mgr.25723) 1012 : cluster 0 pgmap v1054: 99 pgs: 3 remapped+peering, 1 unknown, 7 peering, 88 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:24.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.264373+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:38:24.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.265972+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.266068+0000 osd.2 (osd.2) 1 : cluster 0 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 219'13728 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.267035+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.267752+0000 osd.3 (osd.3) 11 : cluster 0 8.16 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.267903+0000 osd.0 (osd.0) 19 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13220 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.267961+0000 osd.3 (osd.3) 12 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13328 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.268432+0000 osd.3 (osd.3) 13 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13063 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.269920+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.270290+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.270980+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.273551+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.274629+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.275114+0000 osd.6 (osd.6) 182 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13027 2023-10-09T06:38:24.303 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.277574+0000 osd.4 (osd.4) 170 : cluster 0 8.18 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.277609+0000 osd.4 (osd.4) 171 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13347 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.278453+0000 osd.4 (osd.4) 172 : cluster 0 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 217'13235 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.278584+0000 osd.4 (osd.4) 173 : cluster 0 8.19 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.278753+0000 osd.7 (osd.7) 147 : cluster 0 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13349 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.279220+0000 osd.7 (osd.7) 148 : cluster 0 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 221'13484 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.279605+0000 osd.7 (osd.7) 149 : cluster 0 8.11 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13228 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.987050+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.223187+0000 mgr.y (mgr.25723) 1012 : cluster 0 pgmap v1054: 99 pgs: 3 remapped+peering, 1 unknown, 7 peering, 88 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.264373+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:38:24.304 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.265972+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.266068+0000 osd.2 (osd.2) 1 : cluster 0 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 219'13728 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.267035+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.267752+0000 osd.3 (osd.3) 11 : cluster 0 8.16 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.267903+0000 osd.0 (osd.0) 19 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13220 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.267961+0000 osd.3 (osd.3) 12 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13328 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.268432+0000 osd.3 (osd.3) 13 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13063 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.269920+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.270290+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.305 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.270980+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.273551+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.274629+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.275114+0000 osd.6 (osd.6) 182 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13027 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.277574+0000 osd.4 (osd.4) 170 : cluster 0 8.18 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.277609+0000 osd.4 (osd.4) 171 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13347 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.278453+0000 osd.4 (osd.4) 172 : cluster 0 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 217'13235 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.278584+0000 osd.4 (osd.4) 173 : cluster 0 8.19 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.278753+0000 osd.7 (osd.7) 147 : cluster 0 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13349 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.279220+0000 osd.7 (osd.7) 148 : cluster 0 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 221'13484 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.279605+0000 osd.7 (osd.7) 149 : cluster 0 8.11 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13228 2023-10-09T06:38:24.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:24 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.987050+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:38:24.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:24.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:24.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:24.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.223187+0000 mgr.y (mgr.25723) 1012 : cluster 0 pgmap v1054: 99 pgs: 3 remapped+peering, 1 unknown, 7 peering, 88 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.264373+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.265972+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.266068+0000 osd.2 (osd.2) 1 : cluster 0 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 219'13728 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.267035+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.267752+0000 osd.3 (osd.3) 11 : cluster 0 8.16 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13211 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.267903+0000 osd.0 (osd.0) 19 : cluster 0 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13220 2023-10-09T06:38:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.267961+0000 osd.3 (osd.3) 12 : cluster 0 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 219'13328 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.268432+0000 osd.3 (osd.3) 13 : cluster 0 8.8 starting backfill to osd.0 from (0'0,0'0] MAX to 219'13063 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.269920+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.270290+0000 osd.4 (osd.4) 168 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.270980+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.273551+0000 osd.4 (osd.4) 169 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.274629+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e234 with expected crc 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.275114+0000 osd.6 (osd.6) 182 : cluster 0 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 219'13027 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.277574+0000 osd.4 (osd.4) 170 : cluster 0 8.18 starting backfill to osd.7 from (0'0,0'0] MAX to 219'13064 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.277609+0000 osd.4 (osd.4) 171 : cluster 0 8.2 starting backfill to osd.3 from (0'0,0'0] MAX to 219'13347 2023-10-09T06:38:24.570 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.278453+0000 osd.4 (osd.4) 172 : cluster 0 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 217'13235 2023-10-09T06:38:24.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.278584+0000 osd.4 (osd.4) 173 : cluster 0 8.19 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13235 2023-10-09T06:38:24.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.278753+0000 osd.7 (osd.7) 147 : cluster 0 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 219'13349 2023-10-09T06:38:24.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.279220+0000 osd.7 (osd.7) 148 : cluster 0 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 221'13484 2023-10-09T06:38:24.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.279605+0000 osd.7 (osd.7) 149 : cluster 0 8.11 starting backfill to osd.1 from (0'0,0'0] MAX to 219'13228 2023-10-09T06:38:24.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:24 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.987050+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:38:24.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:24.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:24.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:24.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:24.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:24.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:25.056 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:25.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:25.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:25.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.341 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.990115+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.990135+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.992552+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.992731+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.995691+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:23.997528+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.000063+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.990695+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:38:25.342 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.990115+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.990135+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.992552+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.992731+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.995691+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:23.997528+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.000063+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.343 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:25 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.990695+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:38:25.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:25.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:25.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:25.595 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.990115+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.990135+0000 osd.4 (osd.4) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.992552+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.992731+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.995691+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:23.997528+0000 osd.4 (osd.4) 175 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.000063+0000 osd.4 (osd.4) 176 : cluster 3 failed to encode map e235 with expected crc 2023-10-09T06:38:25.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:25 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.990695+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:38:25.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:25.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:25.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:25.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:25.889 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:25.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:25.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:26.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:26.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:26.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:26.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:26.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:26.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:26.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:26.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.992046+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.994000+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.994546+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.996013+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.996249+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:24.997038+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:25.050230+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:26 smithi155 bash[59495]: cluster 2023-10-09T06:38:25.223620+0000 mgr.y (mgr.25723) 1013 : cluster 0 pgmap v1058: 99 pgs: 3 remapped+peering, 1 unknown, 12 peering, 83 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 151 B/s, 0 objects/s recovering 2023-10-09T06:38:26.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:26.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.992046+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.994000+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.994546+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.996013+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.996249+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:24.997038+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:25.050230+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[77299]: cluster 2023-10-09T06:38:25.223620+0000 mgr.y (mgr.25723) 1013 : cluster 0 pgmap v1058: 99 pgs: 3 remapped+peering, 1 unknown, 12 peering, 83 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 151 B/s, 0 objects/s recovering 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.992046+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.994000+0000 osd.4 (osd.4) 177 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.994546+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.996013+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.996249+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:24.997038+0000 osd.4 (osd.4) 178 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.525 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:25.050230+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e236 with expected crc 2023-10-09T06:38:26.526 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:26 smithi125 bash[80002]: cluster 2023-10-09T06:38:25.223620+0000 mgr.y (mgr.25723) 1013 : cluster 0 pgmap v1058: 99 pgs: 3 remapped+peering, 1 unknown, 12 peering, 83 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 151 B/s, 0 objects/s recovering 2023-10-09T06:38:26.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:26.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:26.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:26.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:26.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:26.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:26.944 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:26.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:27.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:27.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:27.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:27.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:27.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:27.315 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:27.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:27.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:27 smithi155 bash[59495]: cluster 2023-10-09T06:38:26.269179+0000 osd.4 (osd.4) 179 : cluster 0 8.18 deep-scrub starts 2023-10-09T06:38:27.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:27 smithi155 bash[59495]: cluster 2023-10-09T06:38:26.285261+0000 osd.4 (osd.4) 180 : cluster 0 8.18 deep-scrub ok 2023-10-09T06:38:27.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:27.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:27.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:27.603 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:38:27.604 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:38:27.604 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:38:27.604 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.586678 2023-10-09T06:38:27.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.5866784832798821 2023-10-09T06:38:27.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:27.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:27 smithi125 bash[77299]: cluster 2023-10-09T06:38:26.269179+0000 osd.4 (osd.4) 179 : cluster 0 8.18 deep-scrub starts 2023-10-09T06:38:27.618 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:27 smithi125 bash[77299]: cluster 2023-10-09T06:38:26.285261+0000 osd.4 (osd.4) 180 : cluster 0 8.18 deep-scrub ok 2023-10-09T06:38:27.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:27 smithi125 bash[80002]: cluster 2023-10-09T06:38:26.269179+0000 osd.4 (osd.4) 179 : cluster 0 8.18 deep-scrub starts 2023-10-09T06:38:27.619 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:27 smithi125 bash[80002]: cluster 2023-10-09T06:38:26.285261+0000 osd.4 (osd.4) 180 : cluster 0 8.18 deep-scrub ok 2023-10-09T06:38:27.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:27.740 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:27.843 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:27.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:27.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:27.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:28.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:28.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:28.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:28.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:28.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:28.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:28.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:28.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:28.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:28.538 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:28 smithi125 bash[77299]: cluster 2023-10-09T06:38:27.224088+0000 mgr.y (mgr.25723) 1014 : cluster 0 pgmap v1059: 99 pgs: 3 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 121 B/s, 2 objects/s recovering 2023-10-09T06:38:28.539 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:28 smithi125 bash[80002]: cluster 2023-10-09T06:38:27.224088+0000 mgr.y (mgr.25723) 1014 : cluster 0 pgmap v1059: 99 pgs: 3 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 121 B/s, 2 objects/s recovering 2023-10-09T06:38:28.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:28 smithi155 bash[59495]: cluster 2023-10-09T06:38:27.224088+0000 mgr.y (mgr.25723) 1014 : cluster 0 pgmap v1059: 99 pgs: 3 remapped+peering, 9 peering, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 121 B/s, 2 objects/s recovering 2023-10-09T06:38:28.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:28.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:28.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:28.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:28.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:28.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:29.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:29.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:29.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:29.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:29.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:29.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:29.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:29.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:29.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:29.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:29.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:29.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:29.799 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:29.821 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:29.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:29.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:29.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:29.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:29.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:30.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:30.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:30.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:30.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:30.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:30.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:30 smithi125 bash[77299]: audit 2023-10-09T06:38:29.171463+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:30.382 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:30 smithi125 bash[77299]: cluster 2023-10-09T06:38:29.224562+0000 mgr.y (mgr.25723) 1015 : cluster 0 pgmap v1060: 99 pgs: 9 peering, 90 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 100 B/s, 1 objects/s recovering 2023-10-09T06:38:30.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:30 smithi125 bash[80002]: audit 2023-10-09T06:38:29.171463+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:30.383 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:30 smithi125 bash[80002]: cluster 2023-10-09T06:38:29.224562+0000 mgr.y (mgr.25723) 1015 : cluster 0 pgmap v1060: 99 pgs: 9 peering, 90 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 100 B/s, 1 objects/s recovering 2023-10-09T06:38:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:30 smithi155 bash[59495]: audit 2023-10-09T06:38:29.171463+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:30 smithi155 bash[59495]: cluster 2023-10-09T06:38:29.224562+0000 mgr.y (mgr.25723) 1015 : cluster 0 pgmap v1060: 99 pgs: 9 peering, 90 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 100 B/s, 1 objects/s recovering 2023-10-09T06:38:30.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:30.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:30.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:30.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:30.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:30.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:30.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:30.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:31.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:31.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:31.241 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:31] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-10-09T06:38:31.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:31.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:31.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:31.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:31.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:31.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:31.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:31.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:31.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:31.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:31.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:32.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:32.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:32.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:32.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:32.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:32.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:32.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:32.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:32.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:32.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:32.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:32 smithi155 bash[59495]: cluster 2023-10-09T06:38:31.225185+0000 mgr.y (mgr.25723) 1016 : cluster 0 pgmap v1061: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 75 B/s, 1 objects/s recovering 2023-10-09T06:38:32.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:32.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:32.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:32.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:32.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:32 smithi125 bash[77299]: cluster 2023-10-09T06:38:31.225185+0000 mgr.y (mgr.25723) 1016 : cluster 0 pgmap v1061: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 75 B/s, 1 objects/s recovering 2023-10-09T06:38:32.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:32 smithi125 bash[80002]: cluster 2023-10-09T06:38:31.225185+0000 mgr.y (mgr.25723) 1016 : cluster 0 pgmap v1061: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 75 B/s, 1 objects/s recovering 2023-10-09T06:38:32.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:32.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:32.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:33.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:33.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:33.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:33.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:33.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:33.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:33.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:33.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:33.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:33.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:33.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:34.084 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:34.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:34.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:34.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:34.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:34.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:34.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:34 smithi155 bash[59495]: cluster 2023-10-09T06:38:33.225768+0000 mgr.y (mgr.25723) 1017 : cluster 0 pgmap v1062: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 65 B/s, 1 objects/s recovering 2023-10-09T06:38:34.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:34 smithi155 bash[59495]: audit 2023-10-09T06:38:33.597533+0000 mon.a (mon.0) 1252 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:34.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:34.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:34.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:34.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:34 smithi125 bash[77299]: cluster 2023-10-09T06:38:33.225768+0000 mgr.y (mgr.25723) 1017 : cluster 0 pgmap v1062: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 65 B/s, 1 objects/s recovering 2023-10-09T06:38:34.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:34 smithi125 bash[77299]: audit 2023-10-09T06:38:33.597533+0000 mon.a (mon.0) 1252 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:34 smithi125 bash[80002]: cluster 2023-10-09T06:38:33.225768+0000 mgr.y (mgr.25723) 1017 : cluster 0 pgmap v1062: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 65 B/s, 1 objects/s recovering 2023-10-09T06:38:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:34 smithi125 bash[80002]: audit 2023-10-09T06:38:33.597533+0000 mon.a (mon.0) 1252 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:34.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:34.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:34.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:34.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:34.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:34.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:34.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:35.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:35.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:35.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:35.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:35.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:35.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:35.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:35.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:35.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:35.584 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:35.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:35.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:35.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:35.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:35.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:35.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:35.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:35.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:36.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:36.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:36.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:36.270 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:36.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:36.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:36 smithi155 bash[59495]: cluster 2023-10-09T06:38:35.226898+0000 mgr.y (mgr.25723) 1018 : cluster 0 pgmap v1063: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:36.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:36.435 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:36 smithi125 bash[77299]: cluster 2023-10-09T06:38:35.226898+0000 mgr.y (mgr.25723) 1018 : cluster 0 pgmap v1063: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:36.436 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:36 smithi125 bash[80002]: cluster 2023-10-09T06:38:35.226898+0000 mgr.y (mgr.25723) 1018 : cluster 0 pgmap v1063: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:36.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.540 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:36.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:36.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:36.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:36.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:36.915 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:36.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:36.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:37.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.049 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:37.076 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:37.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:37.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:37.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:37.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:37.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:37.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:37.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:37.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:37.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:37.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:37.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:37.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:37.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:37.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:38.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:38.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:38.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:38.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:38.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:38.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:38.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:38 smithi155 bash[59495]: cluster 2023-10-09T06:38:37.227532+0000 mgr.y (mgr.25723) 1019 : cluster 0 pgmap v1064: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:38:38.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:38.512 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:38.599 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:38 smithi125 bash[80002]: cluster 2023-10-09T06:38:37.227532+0000 mgr.y (mgr.25723) 1019 : cluster 0 pgmap v1064: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:38:38.599 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:38 smithi125 bash[77299]: cluster 2023-10-09T06:38:37.227532+0000 mgr.y (mgr.25723) 1019 : cluster 0 pgmap v1064: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T06:38:38.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:38.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:38.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.779 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:38.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:38.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:39.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:39.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:39.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:39.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:39.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:39.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:39.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:39.329 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_3' removed 2023-10-09T06:38:39.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:39.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:39.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:39.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:39 smithi125 bash[77299]: audit 2023-10-09T06:38:38.539394+0000 mon.a (mon.0) 1253 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:38:39.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:39 smithi125 bash[77299]: audit 2023-10-09T06:38:39.192350+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:39.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:39 smithi125 bash[80002]: audit 2023-10-09T06:38:38.539394+0000 mon.a (mon.0) 1253 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:38:39.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:39 smithi125 bash[80002]: audit 2023-10-09T06:38:39.192350+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:39.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:39.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:39 smithi155 bash[59495]: audit 2023-10-09T06:38:38.539394+0000 mon.a (mon.0) 1253 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:38:39.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:39 smithi155 bash[59495]: audit 2023-10-09T06:38:39.192350+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:39.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:39.704 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:39.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:39.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:39.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:39.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:39.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:39.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:40.065 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:38:40.066 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:38:40.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:40.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:40.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:40.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:40.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:40.374 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:40.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:40.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:40.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:40.623 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.228147+0000 mgr.y (mgr.25723) 1020 : cluster 0 pgmap v1065: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:40.624 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: audit 2023-10-09T06:38:39.313653+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:38:40.624 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.313749+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:38:40.624 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.315879+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.624 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.803539+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.624 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:40 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.803780+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.228147+0000 mgr.y (mgr.25723) 1020 : cluster 0 pgmap v1065: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: audit 2023-10-09T06:38:39.313653+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.313749+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.315879+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.803539+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.803780+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.228147+0000 mgr.y (mgr.25723) 1020 : cluster 0 pgmap v1065: 99 pgs: 99 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: audit 2023-10-09T06:38:39.313653+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.125:0/149163383' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.313749+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.315879+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.803539+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:40 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.803780+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:40.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:40.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:40.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:40.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:40.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:40.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:41.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:41.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:41.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:41.173 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:41.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:41.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:41.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:41.333 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:41] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-09T06:38:41.341 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:38:41.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:41.433 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:41 smithi155 bash[59495]: cluster 2023-10-09T06:38:39.804235+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:41.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:41 smithi155 bash[59495]: audit 2023-10-09T06:38:40.397153+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.125:0/3539172421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:41 smithi155 bash[59495]: audit 2023-10-09T06:38:40.397520+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:41.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:41.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:41.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[77299]: cluster 2023-10-09T06:38:39.804235+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:41.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[77299]: audit 2023-10-09T06:38:40.397153+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.125:0/3539172421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[77299]: audit 2023-10-09T06:38:40.397520+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[80002]: cluster 2023-10-09T06:38:39.804235+0000 osd.4 (osd.4) 181 : cluster 3 failed to encode map e237 with expected crc 2023-10-09T06:38:41.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[80002]: audit 2023-10-09T06:38:40.397153+0000 mon.c (mon.2) 97 : audit 1 from='client.? 172.21.15.125:0/3539172421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:41 smithi125 bash[80002]: audit 2023-10-09T06:38:40.397520+0000 mon.a (mon.0) 1257 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:41.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:41.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:41.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:41.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:41.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:41.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:42.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:42.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:42.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:42.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:42.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:42.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:42.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: cluster 2023-10-09T06:38:41.228892+0000 mgr.y (mgr.25723) 1021 : cluster 0 pgmap v1067: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:42.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: cluster 2023-10-09T06:38:41.311952+0000 mon.a (mon.0) 1258 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:42.555 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: cluster 2023-10-09T06:38:41.329080+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: audit 2023-10-09T06:38:41.329199+0000 mon.a (mon.0) 1259 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: cluster 2023-10-09T06:38:41.329306+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[77299]: cluster 2023-10-09T06:38:41.331505+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: cluster 2023-10-09T06:38:41.228892+0000 mgr.y (mgr.25723) 1021 : cluster 0 pgmap v1067: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: cluster 2023-10-09T06:38:41.311952+0000 mon.a (mon.0) 1258 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: cluster 2023-10-09T06:38:41.329080+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: audit 2023-10-09T06:38:41.329199+0000 mon.a (mon.0) 1259 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: cluster 2023-10-09T06:38:41.329306+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:38:42.556 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:42 smithi125 bash[80002]: cluster 2023-10-09T06:38:41.331505+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: cluster 2023-10-09T06:38:41.228892+0000 mgr.y (mgr.25723) 1021 : cluster 0 pgmap v1067: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: cluster 2023-10-09T06:38:41.311952+0000 mon.a (mon.0) 1258 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: cluster 2023-10-09T06:38:41.329080+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: audit 2023-10-09T06:38:41.329199+0000 mon.a (mon.0) 1259 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: cluster 2023-10-09T06:38:41.329306+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:38:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:42 smithi155 bash[59495]: cluster 2023-10-09T06:38:41.331505+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:42.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:42.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:42.756 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:38:42.777 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:38:42.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:42.800 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:38:42.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:42.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:38:42.844 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:38:42.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:38:42.888 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:38:42.909 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.910 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.911 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:38:42.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:42.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:42.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:43.018 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:38:43.018 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:38:43.018 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T06:38:43.019 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T06:38:43.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:43.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:43.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:38:43.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:43.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:43.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:43.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:43.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:43.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:43.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:43.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[80002]: cluster 2023-10-09T06:38:42.196184+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[80002]: cluster 2023-10-09T06:38:42.354448+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[80002]: audit 2023-10-09T06:38:42.728463+0000 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.125:0/2490952616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:43.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[77299]: cluster 2023-10-09T06:38:42.196184+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[77299]: cluster 2023-10-09T06:38:42.354448+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:43 smithi125 bash[77299]: audit 2023-10-09T06:38:42.728463+0000 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.125:0/2490952616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:43.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:43 smithi155 bash[59495]: cluster 2023-10-09T06:38:42.196184+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:43 smithi155 bash[59495]: cluster 2023-10-09T06:38:42.354448+0000 osd.4 (osd.4) 182 : cluster 3 failed to encode map e238 with expected crc 2023-10-09T06:38:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:43 smithi155 bash[59495]: audit 2023-10-09T06:38:42.728463+0000 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.125:0/2490952616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:43.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:43.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:43.842 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:43.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:43.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:43.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:44.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:44.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:44.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:44.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:44.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:44.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:44.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:44.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:44 smithi125 bash[77299]: cluster 2023-10-09T06:38:43.229328+0000 mgr.y (mgr.25723) 1022 : cluster 0 pgmap v1069: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:44.467 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:44 smithi125 bash[80002]: cluster 2023-10-09T06:38:43.229328+0000 mgr.y (mgr.25723) 1022 : cluster 0 pgmap v1069: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:44.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:44.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:44 smithi155 bash[59495]: cluster 2023-10-09T06:38:43.229328+0000 mgr.y (mgr.25723) 1022 : cluster 0 pgmap v1069: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:44.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:44.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:44.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:44.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:44.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:44.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:44.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:44.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:45.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:45.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:45.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:45.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:45.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:45.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:45.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:45.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:45.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:45.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:45.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:45.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:45.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:45.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:45.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:45.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:45.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:46.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:46.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:46.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:46.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:46.284 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:46.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:46.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:46 smithi125 bash[77299]: cluster 2023-10-09T06:38:45.229945+0000 mgr.y (mgr.25723) 1023 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:46.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:46 smithi125 bash[80002]: cluster 2023-10-09T06:38:45.229945+0000 mgr.y (mgr.25723) 1023 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:46.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:46.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:46.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:46 smithi155 bash[59495]: cluster 2023-10-09T06:38:45.229945+0000 mgr.y (mgr.25723) 1023 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:46.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:46.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:46.749 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:38:46.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:46.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:46.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:46.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:46.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:47.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:47.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:47.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:47.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:47.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:47.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:47.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:47.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:47.586 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:47.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:47.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:38:47.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:47.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:47.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:47.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:47.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:47.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:48.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:48.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:48.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:48.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:48.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:48.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:48.364 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.0 primary-affinity to 0.586678 (8384482) 2023-10-09T06:38:48.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:48 smithi155 bash[59495]: cluster 2023-10-09T06:38:47.230658+0000 mgr.y (mgr.25723) 1024 : cluster 0 pgmap v1071: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:48.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:48 smithi155 bash[59495]: audit 2023-10-09T06:38:47.428058+0000 mon.a (mon.0) 1262 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]: dispatch 2023-10-09T06:38:48.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:48.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:48.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:48 smithi125 bash[77299]: cluster 2023-10-09T06:38:47.230658+0000 mgr.y (mgr.25723) 1024 : cluster 0 pgmap v1071: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:48.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:48 smithi125 bash[77299]: audit 2023-10-09T06:38:47.428058+0000 mon.a (mon.0) 1262 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]: dispatch 2023-10-09T06:38:48.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:48 smithi125 bash[80002]: cluster 2023-10-09T06:38:47.230658+0000 mgr.y (mgr.25723) 1024 : cluster 0 pgmap v1071: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:48.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:48 smithi125 bash[80002]: audit 2023-10-09T06:38:47.428058+0000 mon.a (mon.0) 1262 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]: dispatch 2023-10-09T06:38:48.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:48.692 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:48.791 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:48.791 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:48.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:48.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:48.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:48.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:49.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:49.072 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:49.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:49.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:49.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:49.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:49.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:49.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:49.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:49.588 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: audit 2023-10-09T06:38:48.350296+0000 mon.a (mon.0) 1263 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]': finished 2023-10-09T06:38:49.588 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: cluster 2023-10-09T06:38:48.350415+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: cluster 2023-10-09T06:38:48.358011+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: cluster 2023-10-09T06:38:48.359524+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: cluster 2023-10-09T06:38:48.453380+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: cluster 2023-10-09T06:38:48.453392+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: audit 2023-10-09T06:38:48.603930+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[80002]: audit 2023-10-09T06:38:48.605283+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: audit 2023-10-09T06:38:48.350296+0000 mon.a (mon.0) 1263 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]': finished 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: cluster 2023-10-09T06:38:48.350415+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:38:49.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: cluster 2023-10-09T06:38:48.358011+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: cluster 2023-10-09T06:38:48.359524+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: cluster 2023-10-09T06:38:48.453380+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: cluster 2023-10-09T06:38:48.453392+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: audit 2023-10-09T06:38:48.603930+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:49.590 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:49 smithi125 bash[77299]: audit 2023-10-09T06:38:48.605283+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:49.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: audit 2023-10-09T06:38:48.350296+0000 mon.a (mon.0) 1263 : audit 1 from='client.? 172.21.15.125:0/661141' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.5866784832798821}]': finished 2023-10-09T06:38:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: cluster 2023-10-09T06:38:48.350415+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:38:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: cluster 2023-10-09T06:38:48.358011+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: cluster 2023-10-09T06:38:48.359524+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: cluster 2023-10-09T06:38:48.453380+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: cluster 2023-10-09T06:38:48.453392+0000 osd.4 (osd.4) 183 : cluster 3 failed to encode map e239 with expected crc 2023-10-09T06:38:49.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: audit 2023-10-09T06:38:48.603930+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:38:49.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:49 smithi155 bash[59495]: audit 2023-10-09T06:38:48.605283+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:49.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:49.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:49.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:49.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:49.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:49.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:50.007 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:50.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:50.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:50.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:50.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:50.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:50.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:50 smithi155 bash[59495]: cluster 2023-10-09T06:38:49.231255+0000 mgr.y (mgr.25723) 1025 : cluster 0 pgmap v1073: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:50.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:50 smithi155 bash[59495]: cluster 2023-10-09T06:38:49.365825+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:38:50.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:50 smithi155 bash[59495]: cluster 2023-10-09T06:38:49.368610+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:50 smithi155 bash[59495]: cluster 2023-10-09T06:38:49.371338+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:50 smithi155 bash[59495]: cluster 2023-10-09T06:38:49.373825+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[77299]: cluster 2023-10-09T06:38:49.231255+0000 mgr.y (mgr.25723) 1025 : cluster 0 pgmap v1073: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[77299]: cluster 2023-10-09T06:38:49.365825+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[77299]: cluster 2023-10-09T06:38:49.368610+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[77299]: cluster 2023-10-09T06:38:49.371338+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[77299]: cluster 2023-10-09T06:38:49.373825+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[80002]: cluster 2023-10-09T06:38:49.231255+0000 mgr.y (mgr.25723) 1025 : cluster 0 pgmap v1073: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:38:50.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[80002]: cluster 2023-10-09T06:38:49.365825+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:38:50.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[80002]: cluster 2023-10-09T06:38:49.368610+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[80002]: cluster 2023-10-09T06:38:49.371338+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:50 smithi125 bash[80002]: cluster 2023-10-09T06:38:49.373825+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e240 with expected crc 2023-10-09T06:38:50.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:50.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:50.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:50.782 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:50.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:50.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:50.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:50.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:50.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:51.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.080 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:51.184 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:51.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:51.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:51.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:51.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:51.640 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:38:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:38:51] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.33.4" 2023-10-09T06:38:51.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:51.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:51.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:51.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:51.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:52.006 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:52.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:52.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:52.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:52.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:52.277 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:52.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:52.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:52.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:52.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:52.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:52.549 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:52 smithi125 bash[77299]: cluster 2023-10-09T06:38:51.232177+0000 mgr.y (mgr.25723) 1026 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:52.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:52 smithi125 bash[80002]: cluster 2023-10-09T06:38:51.232177+0000 mgr.y (mgr.25723) 1026 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:52.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:52 smithi155 bash[59495]: cluster 2023-10-09T06:38:51.232177+0000 mgr.y (mgr.25723) 1026 : cluster 0 pgmap v1075: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:52.668 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:52.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:52.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:52.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:52.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:52.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:52.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:52.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:53.021 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:53.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:53.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:53.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:53.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:53.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:53.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:53.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:53.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:53.488 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:53.591 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:53.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:53.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:53.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:53.868 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:53.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:53.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:54.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:54.029 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:54.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:54.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:54.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:54.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:54.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:54.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:54.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:54.546 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:54 smithi155 bash[59495]: cluster 2023-10-09T06:38:53.232953+0000 mgr.y (mgr.25723) 1027 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:54.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:54.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:54.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:54 smithi125 bash[77299]: cluster 2023-10-09T06:38:53.232953+0000 mgr.y (mgr.25723) 1027 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:54.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:54 smithi125 bash[80002]: cluster 2023-10-09T06:38:53.232953+0000 mgr.y (mgr.25723) 1027 : cluster 0 pgmap v1076: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:54.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:54.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:54.984 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:54.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:54.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:55.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:55.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:55.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:55.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:55.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:55.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:55.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:55.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:55.515 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:38:55.515 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:38:55.516 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T06:38:55.516 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:38:55.516 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:38:55.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T06:38:55.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:55.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:55.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:55.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:55.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:55.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:55.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:55.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:55.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:56.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:56.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:56.152 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:56.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:56.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:56.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:56.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:56.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:56.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:56.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:56.572 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:56 smithi155 bash[59495]: cluster 2023-10-09T06:38:55.233669+0000 mgr.y (mgr.25723) 1028 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:56.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:56.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:56 smithi125 bash[80002]: cluster 2023-10-09T06:38:55.233669+0000 mgr.y (mgr.25723) 1028 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:56.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:56 smithi125 bash[77299]: cluster 2023-10-09T06:38:55.233669+0000 mgr.y (mgr.25723) 1028 : cluster 0 pgmap v1077: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:56.699 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:56.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:56.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:56.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:56.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:56.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:56.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:57.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:57.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:57.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:57.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:57.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:57.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:57.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:57.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:57.632 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:57.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:57.763 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:57.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:57.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:57.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:57.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:57.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:58.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:58.064 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:58.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:58.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:58.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:58.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:58.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:58.446 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:38:58 smithi155 bash[59495]: cluster 2023-10-09T06:38:57.234398+0000 mgr.y (mgr.25723) 1029 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:58.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:58.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:58 smithi125 bash[77299]: cluster 2023-10-09T06:38:57.234398+0000 mgr.y (mgr.25723) 1029 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:58.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:58 smithi125 bash[80002]: cluster 2023-10-09T06:38:57.234398+0000 mgr.y (mgr.25723) 1029 : cluster 0 pgmap v1078: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:38:58.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:58.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:58.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:58.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:58.999 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:59.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:59.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:59.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:59.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:59.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:59.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:59.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:59.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:59.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:59.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:59.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:59.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:59.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:59.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:59.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:59.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:59.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:59.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:59.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:38:59.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:00.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:00.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:00 smithi155 bash[59495]: cluster 2023-10-09T06:38:59.234971+0000 mgr.y (mgr.25723) 1030 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:00.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:00.169 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:00.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:00.259 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:38:59 smithi125 bash[80002]: cluster 2023-10-09T06:38:59.234971+0000 mgr.y (mgr.25723) 1030 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:00.259 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:38:59 smithi125 bash[77299]: cluster 2023-10-09T06:38:59.234971+0000 mgr.y (mgr.25723) 1030 : cluster 0 pgmap v1079: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:00.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:00.294 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:00.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:00.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:00.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:00.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:00.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:00.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:00.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:00.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:00.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:00.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:00.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:00.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:01.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:01.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:01.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:01.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:01.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:01.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:01.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:01.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:01.475 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:01] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.33.4" 2023-10-09T06:39:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:01.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:01.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:01.709 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:01.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:01.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:01.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:01.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:01.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:02.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:02.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:02.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:02.374 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:02 smithi155 bash[59495]: cluster 2023-10-09T06:39:01.235587+0000 mgr.y (mgr.25723) 1031 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:02.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:02.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:02.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:02.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:02.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:02 smithi125 bash[77299]: cluster 2023-10-09T06:39:01.235587+0000 mgr.y (mgr.25723) 1031 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:02.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:02 smithi125 bash[80002]: cluster 2023-10-09T06:39:01.235587+0000 mgr.y (mgr.25723) 1031 : cluster 0 pgmap v1080: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-09T06:39:02.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:02.763 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:02.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:02.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:03.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:03.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:03.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:03.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:03.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:03.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:03.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:03.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:03.417 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:03.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:03.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:03.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:03.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:03.683 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:03.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:03.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:03.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:03.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:03.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:03.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:04.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:04.099 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:39:04.099 INFO:teuthology.orchestra.run.smithi155.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:39:04.099 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:39:04.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:04.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:04.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:04.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:04.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:04.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:04 smithi125 bash[77299]: cluster 2023-10-09T06:39:03.236401+0000 mgr.y (mgr.25723) 1032 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:04.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:04 smithi125 bash[77299]: audit 2023-10-09T06:39:03.597773+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:04.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:04.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:04 smithi155 bash[59495]: cluster 2023-10-09T06:39:03.236401+0000 mgr.y (mgr.25723) 1032 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:04.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:04 smithi155 bash[59495]: audit 2023-10-09T06:39:03.597773+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:04.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:04.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:04.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:04.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:04 smithi125 bash[80002]: cluster 2023-10-09T06:39:03.236401+0000 mgr.y (mgr.25723) 1032 : cluster 0 pgmap v1081: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:04.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:04 smithi125 bash[80002]: audit 2023-10-09T06:39:03.597773+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:04.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:04.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:04.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:04.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:05.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:05.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:05.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:05.245 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:05.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209876 8.10 8:0ac997c0:::benchmark_data_smithi125_176369_object100808:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.311 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928408896999997, 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.526103038, 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209876 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.312 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.313 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372665+0000", 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372672+0000", 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.314 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372684+0000", 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2084e-05 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372694+0000", 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.315 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372742+0000", 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8423e-05 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372752+0000", 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7310000000000005e-06 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.316 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.898696+0000", 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5259440899999999 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.898770+0000", 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3543999999999999e-05 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.317 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209923 8.10 8:0959915d:::benchmark_data_smithi125_176369_object100855:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928370227000002, 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5373261359999999, 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.318 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209923 2023-10-09T06:39:05.319 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.319 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.319 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.319 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.322 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:39:05.322 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.322 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.322 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.322 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.323 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372708+0000", 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372715+0000", 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372719+0000", 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4910000000000001e-06 2023-10-09T06:39:05.324 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372790+0000", 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1585e-05 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0857e-05 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.325 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899097+0000", 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.526296098 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899208+0000", 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111191 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.326 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899512+0000", 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030379499999999999 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906408+0000", 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068957419999999998 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906428+0000", 2023-10-09T06:39:05.327 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0163e-05 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906468+0000", 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9363000000000002e-05 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906654+0000", 2023-10-09T06:39:05.328 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000186866 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909813+0000", 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031590680000000001 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909888+0000", 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4727000000000005e-05 2023-10-09T06:39:05.329 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910031+0000", 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324199999999999 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.330 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209926 8.10 8:0d73ff13:::benchmark_data_smithi125_176369_object100858:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928340301000006, 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5375148360000002, 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.331 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209926 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:39:05.332 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372738+0000", 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.993e-06 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.333 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372747+0000", 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3850000000000008e-06 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372753+0000", 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.879e-06 2023-10-09T06:39:05.334 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372835+0000", 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1921000000000004e-05 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372842+0000", 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5039999999999996e-06 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.335 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901234+0000", 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.528391244 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901444+0000", 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020991599999999999 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.336 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901862+0000", 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041795800000000001 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906442+0000", 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045809750000000002 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.337 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906699+0000", 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000256796 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906785+0000", 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6135000000000001e-05 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.338 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909564+0000", 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027786479999999999 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910082+0000", 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051772999999999999 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.339 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910130+0000", 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8482000000000001e-05 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910250+0000", 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119752 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.340 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209934 8.10 8:0cf8a368:::benchmark_data_smithi125_176369_object100866:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928318919999995, 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378215150000001, 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.341 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209934 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.342 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.343 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372756+0000", 2023-10-09T06:39:05.344 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.345 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.345 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.345 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.345 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372759+0000", 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8830000000000002e-06 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372764+0000", 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0900000000000004e-06 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372772+0000", 2023-10-09T06:39:05.346 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3839999999999999e-06 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372878+0000", 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106623 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372889+0000", 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0725e-05 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.347 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904119+0000", 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5312299970000001 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904293+0000", 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000173952 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.348 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904581+0000", 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000288206 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906670+0000", 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020891759999999999 2023-10-09T06:39:05.349 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906845+0000", 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017475499999999999 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909478+0000", 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026327379999999999 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.350 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909651+0000", 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017286600000000001 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910359+0000", 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070834399999999997 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.351 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910409+0000", 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9749000000000001e-05 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910578+0000", 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016941100000000001 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.352 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209937 8.10 8:0a4d40e2:::benchmark_data_smithi125_176369_object100869:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928293233000005, 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.551965877, 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209937 2023-10-09T06:39:05.353 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.354 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372784+0000", 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6089999999999999e-06 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.355 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372790+0000", 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9510000000000003e-06 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372796+0000", 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.259e-06 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.356 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372923+0000", 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012774100000000001 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372928+0000", 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6020000000000002e-06 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906896+0000", 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5339678559999999 2023-10-09T06:39:05.357 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.907123+0000", 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022763699999999999 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.907470+0000", 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034701399999999998 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.358 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910323+0000", 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028525870000000002 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910619+0000", 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029599100000000001 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.359 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.916236+0000", 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056167999999999999 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920508+0000", 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004272395 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.360 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924535+0000", 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040265850000000001 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924577+0000", 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2577000000000002e-05 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.361 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924748+0000", 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017075599999999999 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209941 8.10 8:09a16055:::benchmark_data_smithi125_176369_object100873:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928274787999996, 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5561017150000001, 2023-10-09T06:39:05.362 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209941 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.363 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.364 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.364 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.364 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372800+0000", 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.365 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372803+0000", 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7140000000000002e-06 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372809+0000", 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8329999999999999e-06 2023-10-09T06:39:05.366 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372815+0000", 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4649999999999999e-06 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.368 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372967+0000", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015147199999999999 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372974+0000", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8720000000000002e-06 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910799+0000", 2023-10-09T06:39:05.369 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378241350000001 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910924+0000", 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012560300000000001 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.911303+0000", 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000379162 2023-10-09T06:39:05.370 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.916296+0000", 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049930189999999996 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920533+0000", 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042367350000000002 2023-10-09T06:39:05.371 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920555+0000", 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1563999999999999e-05 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920609+0000", 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4772999999999999e-05 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.372 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928730+0000", 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081201290000000002 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928768+0000", 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8661000000000001e-05 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.373 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928902+0000", 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013415599999999999 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209952 8.10 8:0d737c89:::benchmark_data_smithi125_176369_object100884:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928242263000001, 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5562271050000001, 2023-10-09T06:39:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209952 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.375 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.376 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372835+0000", 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3640000000000002e-06 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372840+0000", 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:39:05.377 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372845+0000", 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2810000000000002e-06 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373007+0000", 2023-10-09T06:39:05.378 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016230199999999999 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373012+0000", 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.527e-06 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.913584+0000", 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.540572426 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.379 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.913740+0000", 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015646900000000001 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.914050+0000", 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030994799999999999 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.380 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920577+0000", 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0065267800000000003 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920632+0000", 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4620999999999998e-05 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920654+0000", 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1880999999999999e-05 2023-10-09T06:39:05.381 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924525+0000", 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038716390000000001 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928916+0000", 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043907859999999998 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.382 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928950+0000", 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3986999999999998e-05 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929060+0000", 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110287 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.383 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209980 8.10 8:08bfd313:::benchmark_data_smithi125_176369_object100912:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928227735999997, 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5691573399999998, 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209980 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.384 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.385 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372847+0000", 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.386 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372850+0000", 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5100000000000001e-06 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372856+0000", 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.212e-06 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.387 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372863+0000", 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7660000000000001e-06 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373048+0000", 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185415 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373056+0000", 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5480000000000004e-06 2023-10-09T06:39:05.388 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920669+0000", 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.54761273 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920766+0000", 2023-10-09T06:39:05.389 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7446000000000002e-05 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.921042+0000", 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027558299999999998 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928717+0000", 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076757220000000003 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.390 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929109+0000", 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00039157699999999999 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929159+0000", 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9985999999999999e-05 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.391 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929235+0000", 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6007999999999995e-05 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.941816+0000", 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012580780999999999 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.941880+0000", 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3804000000000001e-05 2023-10-09T06:39:05.392 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942005+0000", 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012555800000000001 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209985 8.10 8:09a3bec9:::benchmark_data_smithi125_176369_object100917:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:39:05.393 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928207661000002, 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.579772967, 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209985 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.394 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372867+0000", 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.395 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372870+0000", 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5059999999999998e-06 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372882+0000", 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1892000000000001e-05 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.396 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372892+0000", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0473e-05 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373084+0000", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192086 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373089+0000", 2023-10-09T06:39:05.397 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.566e-06 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922465+0000", 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.549376004 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922643+0000", 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017813599999999999 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.398 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922860+0000", 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021646400000000001 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929145+0000", 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062852359999999996 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929168+0000", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3047000000000001e-05 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942069+0000", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01290088 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952399+0000", 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010330548 2023-10-09T06:39:05.400 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952440+0000", 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.07e-05 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952493+0000", 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3449999999999998e-05 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.401 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952641+0000", 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014738500000000001 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210039 8.10 8:0c2bb4d9:::benchmark_data_smithi125_176369_object100971:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928166388999998, 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5800452730000001, 2023-10-09T06:39:05.402 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210039 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.403 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.404 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372911+0000", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.187e-06 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372915+0000", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2830000000000003e-06 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372919+0000", 2023-10-09T06:39:05.405 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5810000000000001e-06 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373122+0000", 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000203628 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373130+0000", 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3849999999999996e-06 2023-10-09T06:39:05.406 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924795+0000", 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.551665587 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924966+0000", 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017071999999999999 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.407 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.925225+0000", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025892899999999997 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929200+0000", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003974712 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929245+0000", 2023-10-09T06:39:05.408 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4753000000000003e-05 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942110+0000", 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012865639 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952687+0000", 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010576962000000001 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.409 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952779+0000", 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1593000000000002e-05 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952826+0000", 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6751e-05 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.410 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952954+0000", 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012888600000000001 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210147 8.10 8:0a1f5bb6:::benchmark_data_smithi125_176369_object101079:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928153911999999, 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5803002130000001, 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.411 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210147 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.412 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372921+0000", 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.413 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372923+0000", 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7179999999999999e-06 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372926+0000", 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.128e-06 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372929+0000", 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.858e-06 2023-10-09T06:39:05.414 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373158+0000", 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022874200000000001 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373163+0000", 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8729999999999998e-06 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.415 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.926856+0000", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.553693335 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.927033+0000", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017658100000000001 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.927286+0000", 2023-10-09T06:39:05.416 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025329999999999998 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929252+0000", 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019665669999999998 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933820+0000", 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045678740000000004 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.417 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942144+0000", 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083233260000000007 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953010+0000", 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010866752 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.418 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953051+0000", 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0169000000000001e-05 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953093+0000", 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2818000000000002e-05 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953222+0000", 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128405 2023-10-09T06:39:05.419 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210173 8.10 8:0e3f8787:::benchmark_data_smithi125_176369_object101105:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928142383999997, 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5804989599999999, 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.420 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210173 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.421 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372939+0000", 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2609999999999999e-06 2023-10-09T06:39:05.422 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372949+0000", 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7850000000000003e-06 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372955+0000", 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4250000000000003e-06 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.423 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373195+0000", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023943200000000001 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373202+0000", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1439999999999997e-06 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929278+0000", 2023-10-09T06:39:05.424 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5560760010000001 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929364+0000", 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6339000000000001e-05 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929557+0000", 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019301499999999999 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.425 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942023+0000", 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012465385000000001 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942179+0000", 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015596100000000001 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.426 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952230+0000", 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010051480999999999 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953256+0000", 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010258730000000001 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953285+0000", 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9149000000000002e-05 2023-10-09T06:39:05.427 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953324+0000", 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9001999999999997e-05 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953432+0000", 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010805400000000001 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.428 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210227 8.10 8:0feaf1d8:::benchmark_data_smithi125_176369_object101159:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928114769999993, 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5807789350000001, 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.429 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210227 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.430 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372960+0000", 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372963+0000", 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6740000000000001e-06 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.431 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372970+0000", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.764e-06 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372975+0000", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1429999999999997e-06 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373228+0000", 2023-10-09T06:39:05.432 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025312799999999998 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373232+0000", 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3100000000000002e-06 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931483+0000", 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.558250446 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.433 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931571+0000", 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8069e-05 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931810+0000", 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023871799999999999 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.434 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942029+0000", 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010219587 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942215+0000", 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018612699999999999 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952275+0000", 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010060176000000001 2023-10-09T06:39:05.435 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953466+0000", 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001190705 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953591+0000", 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000124697 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.436 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953630+0000", 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9629999999999998e-05 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953740+0000", 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109197 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.437 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210237 8.10 8:0d109371:::benchmark_data_smithi125_176369_object101169:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928095920999993, 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.581022412, 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210237 2023-10-09T06:39:05.438 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.439 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372979+0000", 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372982+0000", 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3690000000000001e-06 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.440 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372988+0000", 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7800000000000003e-06 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372996+0000", 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1999999999999997e-06 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373253+0000", 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025763499999999998 2023-10-09T06:39:05.441 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373257+0000", 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7749999999999999e-06 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933851+0000", 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5605936740000002 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.442 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933991+0000", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013992599999999999 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.934269+0000", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027870999999999999 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942076+0000", 2023-10-09T06:39:05.443 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078070090000000002 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952316+0000", 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010239738999999999 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953497+0000", 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011805979999999999 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.444 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953527+0000", 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0144000000000001e-05 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953773+0000", 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024658699999999999 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.445 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953886+0000", 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112611 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954002+0000", 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116005 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.446 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210258 8.10 8:099246c7:::benchmark_data_smithi125_176369_object101190:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928071204000005, 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.581382085, 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210258 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.447 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.448 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373006+0000", 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2180000000000002e-06 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373010+0000", 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8800000000000001e-06 2023-10-09T06:39:05.449 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373014+0000", 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.391e-06 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373278+0000", 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000264054 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.450 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373283+0000", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5530000000000003e-06 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939405+0000", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5661224279999999 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939560+0000", 2023-10-09T06:39:05.451 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015436000000000001 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939857+0000", 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029764599999999999 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952357+0000", 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012499962999999999 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.452 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952967+0000", 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060953199999999998 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953561+0000", 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00059446399999999995 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.453 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954039+0000", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047749899999999999 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954188+0000", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014903200000000001 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954274+0000", 2023-10-09T06:39:05.454 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5918e-05 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954386+0000", 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011252 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210309 8.10 8:0c45fd6a:::benchmark_data_smithi125_176369_object101241:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:39:05.455 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 72.928049513999994, 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5916112199999999, 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210309 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.456 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:39:05.457 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373028+0000", 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.65e-06 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373036+0000", 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5680000000000002e-06 2023-10-09T06:39:05.458 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373042+0000", 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7710000000000003e-06 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373296+0000", 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025467900000000001 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.459 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373300+0000", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.636e-06 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942243+0000", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5689430880000002 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942373+0000", 2023-10-09T06:39:05.460 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013034900000000001 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942714+0000", 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034058500000000002 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952974+0000", 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010259727 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.461 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954070+0000", 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001095993 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954129+0000", 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9283000000000001e-05 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.462 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954422+0000", 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000292916 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964457+0000", 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010035044 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964511+0000", 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3705000000000002e-05 2023-10-09T06:39:05.463 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964637+0000", 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126637 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218069 8.13 8:cc6ab2b7:::benchmark_data_smithi125_176369_object109001:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:39:05.464 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 58.254506968000001, 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.396654641, 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218069 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.465 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046567+0000", 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:39:05.466 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046574+0000", 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8180000000000004e-06 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046589+0000", 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4353e-05 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.467 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046603+0000", 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4068e-05 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046869+0000", 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00026645999999999999 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.468 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046879+0000", 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5480000000000007e-06 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.435975+0000", 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3890958170000001 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.436075+0000", 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010074800000000001 2023-10-09T06:39:05.469 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.436369+0000", 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029384799999999998 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.442819+0000", 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064500520000000004 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.470 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.442827+0000", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4109999999999999e-06 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443032+0000", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020510299999999999 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443070+0000", 2023-10-09T06:39:05.471 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8039e-05 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443100+0000", 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0386999999999999e-05 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443133+0000", 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2907999999999997e-05 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.472 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443223+0000", 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0001999999999997e-05 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218072 8.13 8:c8fab208:::benchmark_data_smithi125_176369_object109004:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 58.254464775999999, 2023-10-09T06:39:05.473 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3985338650000001, 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218072 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:39:05.474 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046610+0000", 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.475 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046614+0000", 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5619999999999999e-06 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046621+0000", 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0040000000000002e-06 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.476 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046627+0000", 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3790000000000003e-06 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046905+0000", 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027833900000000001 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046913+0000", 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7540000000000006e-06 2023-10-09T06:39:05.477 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438151+0000", 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3912373849999999 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438298+0000", 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014730900000000001 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.478 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438591+0000", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029305599999999998 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443037+0000", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044463310000000004 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443262+0000", 2023-10-09T06:39:05.479 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022511100000000001 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443309+0000", 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6928999999999997e-05 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443348+0000", 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8962999999999999e-05 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.480 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445023+0000", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001674966 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445053+0000", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0052999999999998e-05 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445145+0000", 2023-10-09T06:39:05.481 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1274999999999996e-05 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218081 8.13 8:cee58cb1:::benchmark_data_smithi125_176369_object109013:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 58.254432389999998, 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3989159959999999, 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.482 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218081 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.483 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.484 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046645+0000", 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5780000000000001e-06 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046650+0000", 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7659999999999998e-06 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046654+0000", 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5939999999999998e-06 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.485 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046938+0000", 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028391899999999999 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046942+0000", 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2130000000000002e-06 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.486 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.440649+0000", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3937074489999999 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.440778+0000", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012910299999999999 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.441027+0000", 2023-10-09T06:39:05.487 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024819300000000001 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443236+0000", 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022094219999999999 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443381+0000", 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014532100000000001 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.488 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.444892+0000", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001511091 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.444989+0000", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6335999999999994e-05 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445404+0000", 2023-10-09T06:39:05.489 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041531899999999999 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445446+0000", 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2033000000000002e-05 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445559+0000", 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113003 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.490 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218101 8.13 8:cd996c6e:::benchmark_data_smithi125_176369_object109033:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 58.254419306000003, 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.399543532, 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:39:05.491 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218101 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:39:05.492 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046658+0000", 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.748e-06 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.493 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046661+0000", 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4419999999999998e-06 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046664+0000", 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0259999999999999e-06 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.494 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046954+0000", 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000290296 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046958+0000", 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8399999999999997e-06 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443403+0000", 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.396444805 2023-10-09T06:39:05.495 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443479+0000", 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6097e-05 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443659+0000", 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000180042 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445158+0000", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00149876 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445638+0000", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048019799999999999 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445868+0000", 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022962899999999999 2023-10-09T06:39:05.497 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445905+0000", 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6668000000000002e-05 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445957+0000", 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2611999999999997e-05 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.498 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.446029+0000", 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1662999999999995e-05 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.446200+0000", 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000170946 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:39:05.499 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:39:05.500 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:39:05.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:05.501 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_4' created 2023-10-09T06:39:05.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:05 smithi125 bash[77299]: audit 2023-10-09T06:39:04.469355+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.125:0/2481195240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:05 smithi125 bash[77299]: audit 2023-10-09T06:39:04.469724+0000 mon.a (mon.0) 1269 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:05 smithi125 bash[80002]: audit 2023-10-09T06:39:04.469355+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.125:0/2481195240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.501 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:05 smithi125 bash[80002]: audit 2023-10-09T06:39:04.469724+0000 mon.a (mon.0) 1269 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:05.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:05.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:05 smithi155 bash[59495]: audit 2023-10-09T06:39:04.469355+0000 mon.c (mon.2) 98 : audit 1 from='client.? 172.21.15.125:0/2481195240' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:05 smithi155 bash[59495]: audit 2023-10-09T06:39:04.469724+0000 mon.a (mon.0) 1269 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:39:05.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:05.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:05.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:05.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:05.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:05.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:05.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:06.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:06.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:06.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:06.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:06.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:06.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.236996+0000 mgr.y (mgr.25723) 1033 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:06.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: audit 2023-10-09T06:39:05.307410+0000 mon.a (mon.0) 1270 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:39:06.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.307473+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:39:06.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.309430+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.313 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.314532+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.314675+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.317327+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: cluster 2023-10-09T06:39:05.459367+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.314 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[77299]: audit 2023-10-09T06:39:05.774284+0000 mon.a (mon.0) 1272 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:06.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:06.332 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:39:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:06.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.236996+0000 mgr.y (mgr.25723) 1033 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: audit 2023-10-09T06:39:05.307410+0000 mon.a (mon.0) 1270 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.307473+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.309430+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.314532+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.314675+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.317327+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: cluster 2023-10-09T06:39:05.459367+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.449 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:06 smithi155 bash[59495]: audit 2023-10-09T06:39:05.774284+0000 mon.a (mon.0) 1272 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:06.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:06.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.236996+0000 mgr.y (mgr.25723) 1033 : cluster 0 pgmap v1082: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: audit 2023-10-09T06:39:05.307410+0000 mon.a (mon.0) 1270 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.307473+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.309430+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.314532+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.314675+0000 osd.4 (osd.4) 184 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.317327+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: cluster 2023-10-09T06:39:05.459367+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e241 with expected crc 2023-10-09T06:39:06.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:06 smithi125 bash[80002]: audit 2023-10-09T06:39:05.774284+0000 mon.a (mon.0) 1272 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:06.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:06.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:06.686 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:39:06.708 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:39:06.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:06.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:06.728 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:39:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:06.747 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:39:06.769 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:39:06.791 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:39:06.813 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:39:06.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:06.835 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:39:06.837 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:39:06.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:06.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:06.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:06.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:07.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:07.128 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:07.232 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:07.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:07.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:07.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:07.382 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: cluster 2023-10-09T06:39:06.304160+0000 mon.a (mon.0) 1273 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: audit 2023-10-09T06:39:06.315834+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: cluster 2023-10-09T06:39:06.315927+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: cluster 2023-10-09T06:39:06.317642+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: cluster 2023-10-09T06:39:06.320918+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: cluster 2023-10-09T06:39:06.324209+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.383 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:07 smithi155 bash[59495]: audit 2023-10-09T06:39:06.657142+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.125:0/1961980865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:07.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:07.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:07.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:07.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: cluster 2023-10-09T06:39:06.304160+0000 mon.a (mon.0) 1273 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: audit 2023-10-09T06:39:06.315834+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: cluster 2023-10-09T06:39:06.315927+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: cluster 2023-10-09T06:39:06.317642+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: cluster 2023-10-09T06:39:06.320918+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: cluster 2023-10-09T06:39:06.324209+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[80002]: audit 2023-10-09T06:39:06.657142+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.125:0/1961980865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: cluster 2023-10-09T06:39:06.304160+0000 mon.a (mon.0) 1273 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:07.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: audit 2023-10-09T06:39:06.315834+0000 mon.a (mon.0) 1274 : audit 1 from='client.? 172.21.15.125:0/2148898653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:07.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: cluster 2023-10-09T06:39:06.315927+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:39:07.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: cluster 2023-10-09T06:39:06.317642+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: cluster 2023-10-09T06:39:06.320918+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: cluster 2023-10-09T06:39:06.324209+0000 osd.4 (osd.4) 185 : cluster 3 failed to encode map e242 with expected crc 2023-10-09T06:39:07.642 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:07 smithi125 bash[77299]: audit 2023-10-09T06:39:06.657142+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.125:0/1961980865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:07.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:07.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:07.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:07.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:07.922 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:07.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:07.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:08.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:08.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:08.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:08.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:08.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:08.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:08.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:08.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.237605+0000 mgr.y (mgr.25723) 1034 : cluster 0 pgmap v1085: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.329613+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.609834+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.612127+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.666093+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.475 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:08 smithi155 bash[59495]: cluster 2023-10-09T06:39:07.668932+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:08.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:08.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.237605+0000 mgr.y (mgr.25723) 1034 : cluster 0 pgmap v1085: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:08.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.329613+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:39:08.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.609834+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.612127+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.666093+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[77299]: cluster 2023-10-09T06:39:07.668932+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.237605+0000 mgr.y (mgr.25723) 1034 : cluster 0 pgmap v1085: 89 pgs: 14 unknown, 75 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.329613+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.609834+0000 osd.4 (osd.4) 186 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.606 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.612127+0000 osd.4 (osd.4) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.666093+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:08 smithi125 bash[80002]: cluster 2023-10-09T06:39:07.668932+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:08.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:08.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:08.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:08.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:08.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:08.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:08.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:09.024 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:09.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:09.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T06:39:09.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:09.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:09.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:09 smithi155 bash[59495]: cluster 2023-10-09T06:39:08.058132+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:09 smithi155 bash[59495]: cluster 2023-10-09T06:39:08.058151+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:09.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:09.530 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:09 smithi125 bash[77299]: cluster 2023-10-09T06:39:08.058132+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.531 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:09 smithi125 bash[77299]: cluster 2023-10-09T06:39:08.058151+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:09 smithi125 bash[80002]: cluster 2023-10-09T06:39:08.058132+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.531 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:09 smithi125 bash[80002]: cluster 2023-10-09T06:39:08.058151+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e243 with expected crc 2023-10-09T06:39:09.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:39:09.533 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:09.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:09.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:09.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:09.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:09.837 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:09.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:09.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:09.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:09.979 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:39:09.980 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:39:10.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:10.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:10.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:10.244 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:10.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:10.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:10 smithi155 bash[59495]: cluster 2023-10-09T06:39:09.238365+0000 mgr.y (mgr.25723) 1035 : cluster 0 pgmap v1087: 89 pgs: 9 unknown, 80 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:10.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:10.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:10.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:10.629 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:10 smithi125 bash[77299]: cluster 2023-10-09T06:39:09.238365+0000 mgr.y (mgr.25723) 1035 : cluster 0 pgmap v1087: 89 pgs: 9 unknown, 80 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:10.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:10 smithi125 bash[80002]: cluster 2023-10-09T06:39:09.238365+0000 mgr.y (mgr.25723) 1035 : cluster 0 pgmap v1087: 89 pgs: 9 unknown, 80 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:10.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:10.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:10.752 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:10.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:10.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:10.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:10.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:11.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:11.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:11.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:11.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:11.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:11.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:11.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:11.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:11.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:11.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:39:11.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:39:11.546 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:11] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:39:11.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:11.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:11.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:11.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:11.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:11.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:11.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:11.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:11.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:11.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:12.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:12.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:12.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:12.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:12.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:12.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:12.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:12.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:12 smithi125 bash[77299]: cluster 2023-10-09T06:39:11.239369+0000 mgr.y (mgr.25723) 1036 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:12.527 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:12 smithi125 bash[80002]: cluster 2023-10-09T06:39:11.239369+0000 mgr.y (mgr.25723) 1036 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:12.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:12 smithi155 bash[59495]: cluster 2023-10-09T06:39:11.239369+0000 mgr.y (mgr.25723) 1036 : cluster 0 pgmap v1088: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:12.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:12.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:12.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:12.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:12.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:12.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:12.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:13.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:13.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:13.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:13.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:13.336 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:13.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:13.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:13.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:13 smithi125 bash[77299]: audit 2023-10-09T06:39:12.981993+0000 mon.a (mon.0) 1278 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:13.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:13 smithi125 bash[80002]: audit 2023-10-09T06:39:12.981993+0000 mon.a (mon.0) 1278 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:13.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:13.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:13.616 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:13 smithi155 bash[59495]: audit 2023-10-09T06:39:12.981993+0000 mon.a (mon.0) 1278 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:13.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:13.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:13.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:13.777 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:13.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:13.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:13.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:13.917 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:13.917 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:14.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:14.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:14.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:14.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.178 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:14.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:14.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:14.419 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:14.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:14.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.566 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:14 smithi125 bash[80002]: cluster 2023-10-09T06:39:13.240156+0000 mgr.y (mgr.25723) 1037 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:14.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:14 smithi125 bash[77299]: cluster 2023-10-09T06:39:13.240156+0000 mgr.y (mgr.25723) 1037 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:14.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:14.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:14 smithi155 bash[59495]: cluster 2023-10-09T06:39:13.240156+0000 mgr.y (mgr.25723) 1037 : cluster 0 pgmap v1089: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:14.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:14.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:14.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:14.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:14.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:14.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:15.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:15.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:15.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:15.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:15.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:15.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:15.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:15.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:15.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:15.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:15.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:15.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:15.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:15.876 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:15.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:15.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:15.946 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:15.946 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:16.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:16.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:16.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:16.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:16.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:16.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:16.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:16.413 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:16 smithi155 bash[59495]: cluster 2023-10-09T06:39:15.241089+0000 mgr.y (mgr.25723) 1038 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:16.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:16.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:16.549 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:16 smithi125 bash[77299]: cluster 2023-10-09T06:39:15.241089+0000 mgr.y (mgr.25723) 1038 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:16.550 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:16 smithi125 bash[80002]: cluster 2023-10-09T06:39:15.241089+0000 mgr.y (mgr.25723) 1038 : cluster 0 pgmap v1090: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:16.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:16.671 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:16.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:16.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:16.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:16.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:17.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:17.114 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:17.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:17.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:17.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:17.348 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:17.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:17.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:17.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:17.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:17.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:17.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:17.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:17.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:17.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:17.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:17.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:17.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:18.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:18.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:18.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:18.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:18.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:18.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:18.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:18.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:18.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:18 smithi155 bash[59495]: cluster 2023-10-09T06:39:17.241854+0000 mgr.y (mgr.25723) 1039 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:18.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:18.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:18.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:18.530 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:18 smithi125 bash[80002]: cluster 2023-10-09T06:39:17.241854+0000 mgr.y (mgr.25723) 1039 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:18.530 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:18 smithi125 bash[77299]: cluster 2023-10-09T06:39:17.241854+0000 mgr.y (mgr.25723) 1039 : cluster 0 pgmap v1091: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:18.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:18.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:18.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:18.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:18.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:18.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:18.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:18.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:18.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:18.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:19.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:19.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:19.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:19.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:19.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:19.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:19.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:19.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:18.598773+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:19.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:18.999163+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2023-10-09T06:39:19.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:18.999634+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2023-10-09T06:39:19.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:19.000756+0000 mon.a (mon.0) 1282 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:39:19.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:19.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:19.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:19.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:18.598773+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:18.999163+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2023-10-09T06:39:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:18.999634+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2023-10-09T06:39:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:19.000756+0000 mon.a (mon.0) 1282 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:39:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:19.216989+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:19.237271+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:19 smithi155 bash[59495]: audit 2023-10-09T06:39:19.245796+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.684 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:19.216989+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.684 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:19.237271+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[77299]: audit 2023-10-09T06:39:19.245796+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:18.598773+0000 mon.a (mon.0) 1279 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:18.999163+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]: dispatch 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:18.999634+0000 mon.a (mon.0) 1281 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:19.000756+0000 mon.a (mon.0) 1282 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:19.216989+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:19.237271+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.686 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:19 smithi125 bash[80002]: audit 2023-10-09T06:39:19.245796+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:19.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:19.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:19.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:19.862 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:19.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:19.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:19.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:20.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:20.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:20.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:20.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:20.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:20.509 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:20 smithi155 bash[59495]: cluster 2023-10-09T06:39:19.242699+0000 mgr.y (mgr.25723) 1040 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:20.509 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:20 smithi155 bash[59495]: audit 2023-10-09T06:39:20.003071+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2023-10-09T06:39:20.509 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:20 smithi155 bash[59495]: audit 2023-10-09T06:39:20.003179+0000 mon.a (mon.0) 1287 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2023-10-09T06:39:20.509 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:20 smithi155 bash[59495]: audit 2023-10-09T06:39:20.003276+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:39:20.509 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:20 smithi155 bash[59495]: cluster 2023-10-09T06:39:20.003370+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:39:20.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:20.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[77299]: cluster 2023-10-09T06:39:19.242699+0000 mgr.y (mgr.25723) 1040 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:20.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[77299]: audit 2023-10-09T06:39:20.003071+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2023-10-09T06:39:20.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[77299]: audit 2023-10-09T06:39:20.003179+0000 mon.a (mon.0) 1287 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[77299]: audit 2023-10-09T06:39:20.003276+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[77299]: cluster 2023-10-09T06:39:20.003370+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[80002]: cluster 2023-10-09T06:39:19.242699+0000 mgr.y (mgr.25723) 1040 : cluster 0 pgmap v1092: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[80002]: audit 2023-10-09T06:39:20.003071+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 0]}]': finished 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[80002]: audit 2023-10-09T06:39:20.003179+0000 mon.a (mon.0) 1287 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[80002]: audit 2023-10-09T06:39:20.003276+0000 mon.a (mon.0) 1288 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-09T06:39:20.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:20 smithi125 bash[80002]: cluster 2023-10-09T06:39:20.003370+0000 mon.a (mon.0) 1289 : cluster 0 osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:39:20.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:20.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:20.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:20.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:20.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:20.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:20.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:21.036 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:21.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:21.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:21.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:21.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:21.311 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:21.342 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:21.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:21.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:21.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:21.600 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:21] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-10-09T06:39:21.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:21.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:21.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:21.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:21.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:21.874 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:21.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: cluster 2023-10-09T06:39:20.004302+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: cluster 2023-10-09T06:39:20.004542+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: audit 2023-10-09T06:39:20.593625+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: audit 2023-10-09T06:39:20.600601+0000 mon.a (mon.0) 1291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.010373+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:39:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: audit 2023-10-09T06:39:21.076902+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:39:21.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: audit 2023-10-09T06:39:21.078324+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:39:21.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: audit 2023-10-09T06:39:21.086402+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:21 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.243378+0000 mgr.y (mgr.25723) 1041 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:21.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: cluster 2023-10-09T06:39:20.004302+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: cluster 2023-10-09T06:39:20.004542+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: audit 2023-10-09T06:39:20.593625+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: audit 2023-10-09T06:39:20.600601+0000 mon.a (mon.0) 1291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.010373+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: audit 2023-10-09T06:39:21.076902+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: audit 2023-10-09T06:39:21.078324+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: audit 2023-10-09T06:39:21.086402+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.243378+0000 mgr.y (mgr.25723) 1041 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: cluster 2023-10-09T06:39:20.004302+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: cluster 2023-10-09T06:39:20.004542+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e244 with expected crc 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: audit 2023-10-09T06:39:20.593625+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: audit 2023-10-09T06:39:20.600601+0000 mon.a (mon.0) 1291 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.010373+0000 mon.a (mon.0) 1292 : cluster 0 osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: audit 2023-10-09T06:39:21.076902+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: audit 2023-10-09T06:39:21.078324+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:39:21.907 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: audit 2023-10-09T06:39:21.086402+0000 mon.a (mon.0) 1295 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:39:21.908 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:21 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.243378+0000 mgr.y (mgr.25723) 1041 : cluster 0 pgmap v1095: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:21.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:22.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:22.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:22.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:22.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:22.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:22.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:22.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:22.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:22.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:22.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:22.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:22.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:22.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:22.679 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:22.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:22.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:22.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.012179+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.012268+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.015549+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[77299]: cluster 2023-10-09T06:39:21.061965+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.012179+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.012268+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.015549+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:22 smithi125 bash[80002]: cluster 2023-10-09T06:39:21.061965+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:22 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.012179+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:22 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.012268+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:22 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.015549+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:22 smithi155 bash[59495]: cluster 2023-10-09T06:39:21.061965+0000 osd.4 (osd.4) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-09T06:39:22.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:22.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:22.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:23.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:23.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:23.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:23.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:23.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:23.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:23.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:23.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:23.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:23.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:23.754 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:23 smithi155 bash[59495]: cluster 2023-10-09T06:39:23.244289+0000 mgr.y (mgr.25723) 1042 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:23.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:23.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:23.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:23 smithi125 bash[77299]: cluster 2023-10-09T06:39:23.244289+0000 mgr.y (mgr.25723) 1042 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:23.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:23 smithi125 bash[80002]: cluster 2023-10-09T06:39:23.244289+0000 mgr.y (mgr.25723) 1042 : cluster 0 pgmap v1096: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:23.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:23.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:24.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:24.046 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:24.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:24.182 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:24.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:24.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:24.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:24.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:24.455 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:24.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:24.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:24.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:24.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:24.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:24.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:24.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:24.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:24.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:24.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:24.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:25.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:25.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:25.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:25.250 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:25.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:25.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:25.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:25.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:25.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:25.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:25.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:25.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:25.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:25.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:25.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:25.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:26.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:26.050 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:26.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:26.188 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:26.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:26.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:26.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:26.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:26.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:26.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:26.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:26 smithi125 bash[77299]: cluster 2023-10-09T06:39:25.245276+0000 mgr.y (mgr.25723) 1043 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:26.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:26 smithi125 bash[80002]: cluster 2023-10-09T06:39:25.245276+0000 mgr.y (mgr.25723) 1043 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:26.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:26 smithi155 bash[59495]: cluster 2023-10-09T06:39:25.245276+0000 mgr.y (mgr.25723) 1043 : cluster 0 pgmap v1097: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:26.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:26.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:26.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:26.824 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:26.824 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:26.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:26.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:26.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:26.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:26.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:27.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:27.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:27.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:27.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:27.323 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T06:39:27.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:27.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:27.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:27 smithi155 bash[59495]: audit 2023-10-09T06:39:26.883564+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:27.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:27.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:27.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:27.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:27.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:27 smithi125 bash[80002]: audit 2023-10-09T06:39:26.883564+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:27.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:27 smithi125 bash[77299]: audit 2023-10-09T06:39:26.883564+0000 mon.a (mon.0) 1296 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:27.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:27.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:27.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:27.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:27.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:27.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:28.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:28.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:28.143 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:28.179 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:28.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:28.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:28.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:28.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:28.424 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:28 smithi155 bash[59495]: cluster 2023-10-09T06:39:27.245827+0000 mgr.y (mgr.25723) 1044 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:28.424 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:28 smithi155 bash[59495]: audit 2023-10-09T06:39:27.312108+0000 mon.a (mon.0) 1297 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:28.424 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:28 smithi155 bash[59495]: cluster 2023-10-09T06:39:27.312168+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:39:28.424 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:28 smithi155 bash[59495]: cluster 2023-10-09T06:39:27.314919+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T06:39:28.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:28.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[77299]: cluster 2023-10-09T06:39:27.245827+0000 mgr.y (mgr.25723) 1044 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[77299]: audit 2023-10-09T06:39:27.312108+0000 mon.a (mon.0) 1297 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[77299]: cluster 2023-10-09T06:39:27.312168+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[77299]: cluster 2023-10-09T06:39:27.314919+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[80002]: cluster 2023-10-09T06:39:27.245827+0000 mgr.y (mgr.25723) 1044 : cluster 0 pgmap v1098: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:28.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[80002]: audit 2023-10-09T06:39:27.312108+0000 mon.a (mon.0) 1297 : audit 1 from='client.? 172.21.15.125:0/226642007' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:28.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[80002]: cluster 2023-10-09T06:39:27.312168+0000 mon.a (mon.0) 1298 : cluster 0 osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:39:28.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:28 smithi125 bash[80002]: cluster 2023-10-09T06:39:27.314919+0000 osd.4 (osd.4) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-09T06:39:28.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:28.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:28.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:28.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:28.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:28.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:28.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:28.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:28.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:29.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:29.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:29.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:29.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:29.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:29.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:29.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:29.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:29.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:29.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:29.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:29.756 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:29.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:29.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:29.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:29.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:30.021 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:39:30.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:30.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:30 smithi125 bash[77299]: audit 2023-10-09T06:39:29.043970+0000 mon.a (mon.0) 1299 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:30.135 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:30 smithi125 bash[77299]: cluster 2023-10-09T06:39:29.246532+0000 mgr.y (mgr.25723) 1045 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:30.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:30 smithi125 bash[80002]: audit 2023-10-09T06:39:29.043970+0000 mon.a (mon.0) 1299 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:30.135 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:30 smithi125 bash[80002]: cluster 2023-10-09T06:39:29.246532+0000 mgr.y (mgr.25723) 1045 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:30.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:30.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:30.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:30 smithi155 bash[59495]: audit 2023-10-09T06:39:29.043970+0000 mon.a (mon.0) 1299 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:30 smithi155 bash[59495]: cluster 2023-10-09T06:39:29.246532+0000 mgr.y (mgr.25723) 1045 : cluster 0 pgmap v1100: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:30.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:30.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:30.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:30.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:30.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:30.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:30.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:39:30.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:30.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:30.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:31.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: cluster 2023-10-09T06:39:29.999710+0000 mon.a (mon.0) 1300 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:31.042 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: audit 2023-10-09T06:39:30.010903+0000 mon.a (mon.0) 1301 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: cluster 2023-10-09T06:39:30.010981+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: cluster 2023-10-09T06:39:30.011881+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: cluster 2023-10-09T06:39:30.315118+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[77299]: cluster 2023-10-09T06:39:30.318018+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: cluster 2023-10-09T06:39:29.999710+0000 mon.a (mon.0) 1300 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: audit 2023-10-09T06:39:30.010903+0000 mon.a (mon.0) 1301 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: cluster 2023-10-09T06:39:30.010981+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: cluster 2023-10-09T06:39:30.011881+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.043 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: cluster 2023-10-09T06:39:30.315118+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.044 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[80002]: cluster 2023-10-09T06:39:30.318018+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:31.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:31.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:31.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:31.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:31.262 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:39:31.285 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:39:31.310 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:39:31.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:31.332 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: cluster 2023-10-09T06:39:29.999710+0000 mon.a (mon.0) 1300 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: audit 2023-10-09T06:39:30.010903+0000 mon.a (mon.0) 1301 : audit 1 from='client.? 172.21.15.125:0/493247458' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: cluster 2023-10-09T06:39:30.010981+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: cluster 2023-10-09T06:39:30.011881+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: cluster 2023-10-09T06:39:30.315118+0000 osd.4 (osd.4) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.341 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:31 smithi155 bash[59495]: cluster 2023-10-09T06:39:30.318018+0000 osd.4 (osd.4) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:31.355 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:39:31.359 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:31.378 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:39:31.401 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:39:31.429 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:39:31.430 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.430 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.431 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:39:31.463 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:31.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:31] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-10-09T06:39:31.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:31.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:31.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:31.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:31.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:31.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:31.738 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T06:39:31.738 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:39:31.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:39:31.741 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:39:31.741 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:39:31.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:31.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:31.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:31.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:31.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:31.938 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:39:31.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:39:31.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_199533 2023-10-09T06:39:31.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:39:31.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:39:31.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:32.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:32.037 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[80002]: cluster 2023-10-09T06:39:31.090746+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[80002]: cluster 2023-10-09T06:39:31.163302+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[80002]: audit 2023-10-09T06:39:31.235530+0000 mon.a (mon.0) 1303 : audit 0 from='client.? 172.21.15.125:0/834929167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:32.038 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[80002]: cluster 2023-10-09T06:39:31.246925+0000 mgr.y (mgr.25723) 1046 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:32.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:32.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:32.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:32.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:32.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[77299]: cluster 2023-10-09T06:39:31.090746+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[77299]: cluster 2023-10-09T06:39:31.163302+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[77299]: audit 2023-10-09T06:39:31.235530+0000 mon.a (mon.0) 1303 : audit 0 from='client.? 172.21.15.125:0/834929167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:32.292 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:32 smithi125 bash[77299]: cluster 2023-10-09T06:39:31.246925+0000 mgr.y (mgr.25723) 1046 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:32 smithi155 bash[59495]: cluster 2023-10-09T06:39:31.090746+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:32 smithi155 bash[59495]: cluster 2023-10-09T06:39:31.163302+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-09T06:39:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:32 smithi155 bash[59495]: audit 2023-10-09T06:39:31.235530+0000 mon.a (mon.0) 1303 : audit 0 from='client.? 172.21.15.125:0/834929167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:32.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:32 smithi155 bash[59495]: cluster 2023-10-09T06:39:31.246925+0000 mgr.y (mgr.25723) 1046 : cluster 0 pgmap v1102: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-09T06:39:32.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:32.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:32.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:32.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:32.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:32.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:32.774 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:32.805 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:32.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:32.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:32.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:32.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:32.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 1976 1960 122.493 122.5 0.00413565 0.00813155 2023-10-09T06:39:33.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:33.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:33.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:33.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:33.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:33.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:33.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:33.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:33.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:33.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:33.622 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:33.726 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:33.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:33.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:33.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:33.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:33.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 4680 4664 145.737 169 0.00250517 0.00685213 2023-10-09T06:39:33.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:34.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:34.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:34.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:39:34.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:34.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:34.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:34.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:34.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:34.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:34.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:34 smithi125 bash[77299]: cluster 2023-10-09T06:39:33.247465+0000 mgr.y (mgr.25723) 1047 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 30 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 3.2 MiB/s wr, 50 op/s 2023-10-09T06:39:34.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:34 smithi125 bash[77299]: audit 2023-10-09T06:39:33.598596+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:34.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:34 smithi125 bash[80002]: cluster 2023-10-09T06:39:33.247465+0000 mgr.y (mgr.25723) 1047 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 30 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 3.2 MiB/s wr, 50 op/s 2023-10-09T06:39:34.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:34 smithi125 bash[80002]: audit 2023-10-09T06:39:33.598596+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:34.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:34 smithi155 bash[59495]: cluster 2023-10-09T06:39:33.247465+0000 mgr.y (mgr.25723) 1047 : cluster 0 pgmap v1103: 89 pgs: 89 active+clean; 30 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 3.2 MiB/s wr, 50 op/s 2023-10-09T06:39:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:34 smithi155 bash[59495]: audit 2023-10-09T06:39:33.598596+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:34.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:34.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:34.836 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:34.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:34.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 7159 7143 148.799 154.938 0.00557621 0.00670743 2023-10-09T06:39:34.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:34.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:35.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:35.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:35.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:35.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:39:35.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:35.372 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:35.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:35.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:35.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:35.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:35.639 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:35.667 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:35.668 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:35.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:35.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:35.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:35.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:35.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:35.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:35.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:35.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 9422 9406 146.956 141.438 0.00318796 0.00679386 2023-10-09T06:39:36.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:36.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:36.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:36.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:36.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:36.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:36.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:36.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:36.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:36.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:36 smithi155 bash[59495]: cluster 2023-10-09T06:39:35.248425+0000 mgr.y (mgr.25723) 1048 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 235 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 460 op/s 2023-10-09T06:39:36.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:36.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:36.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:36 smithi125 bash[77299]: cluster 2023-10-09T06:39:35.248425+0000 mgr.y (mgr.25723) 1048 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 235 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 460 op/s 2023-10-09T06:39:36.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:36 smithi125 bash[80002]: cluster 2023-10-09T06:39:35.248425+0000 mgr.y (mgr.25723) 1048 : cluster 0 pgmap v1104: 89 pgs: 89 active+clean; 235 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 460 op/s 2023-10-09T06:39:36.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:36.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:36.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:36.844 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:36.939 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 11740 11724 146.537 144.875 0.00739599 0.00681443 2023-10-09T06:39:36.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:36.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:36.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:37.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:37.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:37.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:37.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:37.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:37.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:37.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:37.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:37.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:37.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:37.742 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:37.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:37.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:37.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:37.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:37.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 13805 13789 143.622 129.062 0.00368733 0.00690512 2023-10-09T06:39:38.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:38.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:38.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:38.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:38.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:38.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:38.336 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:38 smithi125 bash[77299]: cluster 2023-10-09T06:39:37.248921+0000 mgr.y (mgr.25723) 1049 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 321 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s 2023-10-09T06:39:38.336 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:38 smithi125 bash[80002]: cluster 2023-10-09T06:39:37.248921+0000 mgr.y (mgr.25723) 1049 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 321 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s 2023-10-09T06:39:38.405 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:38.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:38.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:38.545 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:38 smithi155 bash[59495]: cluster 2023-10-09T06:39:37.248921+0000 mgr.y (mgr.25723) 1049 : cluster 0 pgmap v1105: 89 pgs: 89 active+clean; 321 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 32 MiB/s wr, 508 op/s 2023-10-09T06:39:38.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:38.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:38.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:38.724 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:38.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:38.822 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:38.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:38.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 15758 15742 140.541 122.062 0.00634727 0.00710736 2023-10-09T06:39:38.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:38.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:39.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:39.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:39.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:39.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:39.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:39.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:39.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:39.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:39.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:39.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:39.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:39.749 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:39.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:39.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:39.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:39.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 17899 17883 139.698 133.812 0.00458813 0.00715114 2023-10-09T06:39:39.978 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:40.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:40.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:40.127 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:40 smithi125 bash[77299]: cluster 2023-10-09T06:39:39.249522+0000 mgr.y (mgr.25723) 1050 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 548 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:39:40.127 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:40 smithi125 bash[80002]: cluster 2023-10-09T06:39:39.249522+0000 mgr.y (mgr.25723) 1050 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 548 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:39:40.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:40.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:40 smithi155 bash[59495]: cluster 2023-10-09T06:39:39.249522+0000 mgr.y (mgr.25723) 1050 : cluster 0 pgmap v1106: 89 pgs: 89 active+clean; 548 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:39:40.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:40.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:40.400 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:40.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:40.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:40.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:40.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:40.929 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:40.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:40.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 19864 19848 137.821 122.812 0.00658428 0.007248 2023-10-09T06:39:41.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:41.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:41.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:41.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:41.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:41] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.33.4" 2023-10-09T06:39:41.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:41.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:41.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:41.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.848 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:41.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 21532 21516 134.463 104.25 0.00393393 0.00743016 2023-10-09T06:39:41.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:41.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:42.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:42.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:42.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:42.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:42.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:42.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:42.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:42.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:42.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:42.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:42.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:42 smithi155 bash[59495]: cluster 2023-10-09T06:39:41.250183+0000 mgr.y (mgr.25723) 1051 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 932 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:39:42.632 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:42 smithi125 bash[77299]: cluster 2023-10-09T06:39:41.250183+0000 mgr.y (mgr.25723) 1051 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 932 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:39:42.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:42 smithi125 bash[80002]: cluster 2023-10-09T06:39:41.250183+0000 mgr.y (mgr.25723) 1051 : cluster 0 pgmap v1107: 89 pgs: 89 active+clean; 932 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:39:42.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:42.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:42.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:42.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:42.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:42.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:42.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:42.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 23177 23161 131.584 102.812 0.00335348 0.00759432 2023-10-09T06:39:43.011 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:43.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:43.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:43.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:43.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:43.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:43.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:43.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:43.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:43.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:43.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:43.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:43.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:43.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:43.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:43.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:43.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:43.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:43.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 24934 24918 129.77 109.812 0.0058629 0.00769879 2023-10-09T06:39:43.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:43.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:44.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:44.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:44.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:44.224 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:44.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:44.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:44 smithi125 bash[77299]: cluster 2023-10-09T06:39:43.250655+0000 mgr.y (mgr.25723) 1052 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:39:44.338 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:44 smithi125 bash[80002]: cluster 2023-10-09T06:39:43.250655+0000 mgr.y (mgr.25723) 1052 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:39:44.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:44.460 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:44.490 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:44.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:44.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:44 smithi155 bash[59495]: cluster 2023-10-09T06:39:43.250655+0000 mgr.y (mgr.25723) 1052 : cluster 0 pgmap v1108: 89 pgs: 89 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T06:39:44.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:44.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:44.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:44.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:44.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:44.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:44.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 26473 26457 127.185 96.1875 0.00649907 0.00785646 2023-10-09T06:39:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:44.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:45.043 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:45.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:45.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:45.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:45.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:45.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:45.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:45.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:45.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:45.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:45.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:45.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:45.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:45.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:45.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:45.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:45.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:45.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 27995 27979 124.895 95.125 0.00804726 0.00799886 2023-10-09T06:39:45.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:45.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:45.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:46.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:46.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:46.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:46.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:46.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:46.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:46.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:46 smithi155 bash[59495]: cluster 2023-10-09T06:39:45.251304+0000 mgr.y (mgr.25723) 1053 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T06:39:46.656 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:46 smithi125 bash[77299]: cluster 2023-10-09T06:39:45.251304+0000 mgr.y (mgr.25723) 1053 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T06:39:46.657 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:46 smithi125 bash[80002]: cluster 2023-10-09T06:39:45.251304+0000 mgr.y (mgr.25723) 1053 : cluster 0 pgmap v1109: 89 pgs: 89 active+clean; 1.5 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T06:39:46.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:46.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.785 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:46.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:46.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:46.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:46.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 29203 29187 121.601 75.5 0.00555994 0.00821737 2023-10-09T06:39:47.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:47.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:47.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:47.178 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:47.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:47.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:47.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:47.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:47.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:47.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:47.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:47.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:47.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:47.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:47.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 30921 30905 120.711 107.375 0.0072973 0.00827604 2023-10-09T06:39:47.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:47.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:47.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:48.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:48.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:48.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:48.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:48.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:48.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:48.374 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:48 smithi125 bash[77299]: cluster 2023-10-09T06:39:47.251872+0000 mgr.y (mgr.25723) 1054 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:39:48.374 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:48 smithi125 bash[80002]: cluster 2023-10-09T06:39:47.251872+0000 mgr.y (mgr.25723) 1054 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:39:48.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:48.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:48.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:48.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:48.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:48.639 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:48 smithi155 bash[59495]: cluster 2023-10-09T06:39:47.251872+0000 mgr.y (mgr.25723) 1054 : cluster 0 pgmap v1110: 89 pgs: 89 active+clean; 1.5 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:39:48.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:48.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:48.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:48.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:48.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:48.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:48.940 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 32286 32270 118.629 85.3125 0.00591896 0.0084229 2023-10-09T06:39:49.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:49.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:49.289 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:49.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:49.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:49.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:49.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:49.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:49 smithi155 bash[59495]: audit 2023-10-09T06:39:48.598783+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:49.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:49.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:49 smithi125 bash[77299]: audit 2023-10-09T06:39:48.598783+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:49.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:49 smithi125 bash[80002]: audit 2023-10-09T06:39:48.598783+0000 mon.a (mon.0) 1305 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:49.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:49.710 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:49.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:49.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 15 33887 33872 117.6 100.125 0.0077177 0.00849619 2023-10-09T06:39:49.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:49.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:49.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:50.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:50.114 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:50.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:50.329 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:50 smithi125 bash[77299]: cluster 2023-10-09T06:39:49.252387+0000 mgr.y (mgr.25723) 1055 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T06:39:50.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:50.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:50.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:50.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:50 smithi155 bash[59495]: cluster 2023-10-09T06:39:49.252387+0000 mgr.y (mgr.25723) 1055 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T06:39:50.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:50.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:50.618 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:50 smithi125 bash[80002]: cluster 2023-10-09T06:39:49.252387+0000 mgr.y (mgr.25723) 1055 : cluster 0 pgmap v1111: 89 pgs: 89 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-09T06:39:50.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:50.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:50.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:50.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:50.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:50.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:50.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 35423 35407 116.46 95.9375 0.00744329 0.00858021 2023-10-09T06:39:51.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:51.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:51.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:51.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:51.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:51.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:51.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:51.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:51.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:51.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:51.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:51.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:51.688 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:51.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:51.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:51.722 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:39:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:39:51] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-10-09T06:39:51.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:51.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:51.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:39:51.935287+0000 min lat: 0.00126636 max lat: 0.342197 avg lat: 0.00870929 2023-10-09T06:39:51.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:39:51.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 36736 36720 114.739 82.0625 0.00370215 0.00870929 2023-10-09T06:39:51.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:51.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:51.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:52.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:52.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:52.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:52.253 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:52.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:52.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:52.496 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:52.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:52 smithi155 bash[59495]: cluster 2023-10-09T06:39:51.253180+0000 mgr.y (mgr.25723) 1056 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T06:39:52.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:52.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:52.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:52.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:52 smithi125 bash[77299]: cluster 2023-10-09T06:39:51.253180+0000 mgr.y (mgr.25723) 1056 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T06:39:52.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:52 smithi125 bash[80002]: cluster 2023-10-09T06:39:51.253180+0000 mgr.y (mgr.25723) 1056 : cluster 0 pgmap v1112: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-10-09T06:39:52.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:52.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:52.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:52.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:52.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:52.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 37956 37940 112.906 76.25 0.0128386 0.00885105 2023-10-09T06:39:53.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:53.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:53.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:53.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:53.272 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:53.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:53.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:53.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:53.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:53.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:53.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:53.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:53.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 39468 39452 112.069 94.5 0.00884419 0.00891716 2023-10-09T06:39:53.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:53.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:54.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:54.081 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:54.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:54.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:54.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:54.449 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:54.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:54.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.613 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:54 smithi125 bash[77299]: cluster 2023-10-09T06:39:53.253710+0000 mgr.y (mgr.25723) 1057 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:54.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:54 smithi125 bash[80002]: cluster 2023-10-09T06:39:53.253710+0000 mgr.y (mgr.25723) 1057 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:54.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:54 smithi155 bash[59495]: cluster 2023-10-09T06:39:53.253710+0000 mgr.y (mgr.25723) 1057 : cluster 0 pgmap v1113: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:54.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:54.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:54.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:54.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:54.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 40687 40671 110.509 76.1875 0.0100795 0.00904309 2023-10-09T06:39:54.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:55.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:55.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:55.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:55.291 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:55.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:55.396 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:55.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:55.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:55.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:55.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:55.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:55.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:55.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 42079 42063 109.529 87 0.00659665 0.00912303 2023-10-09T06:39:55.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:55.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:56.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:56.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:56.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:56.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:56.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:56.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:56.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:56.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:56.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:56.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:56 smithi155 bash[59495]: cluster 2023-10-09T06:39:55.254286+0000 mgr.y (mgr.25723) 1058 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:56.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:56.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:56 smithi125 bash[77299]: cluster 2023-10-09T06:39:55.254286+0000 mgr.y (mgr.25723) 1058 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:56.655 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:56 smithi125 bash[80002]: cluster 2023-10-09T06:39:55.254286+0000 mgr.y (mgr.25723) 1058 : cluster 0 pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:39:56.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:56.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:56.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:56.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:56.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:56.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 43371 43355 108.377 80.75 0.00465742 0.00922113 2023-10-09T06:39:56.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:57.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:57.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:57.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:57.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:57.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:57.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:57.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:57.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:57.551 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:57.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:57.680 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:57.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:57.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:57.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:57.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 44753 44737 107.531 86.375 0.00434122 0.00929345 2023-10-09T06:39:57.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:57.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:58.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:58.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:58.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:58.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:58.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:58.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:58.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:58.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:58.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:58.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:58.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:58.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:39:58 smithi155 bash[59495]: cluster 2023-10-09T06:39:57.254767+0000 mgr.y (mgr.25723) 1059 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:39:58.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:39:58 smithi125 bash[77299]: cluster 2023-10-09T06:39:57.254767+0000 mgr.y (mgr.25723) 1059 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:39:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:39:58 smithi125 bash[80002]: cluster 2023-10-09T06:39:57.254767+0000 mgr.y (mgr.25723) 1059 : cluster 0 pgmap v1115: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:39:58.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:58.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:58.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:58.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:58.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:58.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 45975 45959 106.377 76.375 0.0105378 0.0093946 2023-10-09T06:39:58.975 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:59.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:59.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:59.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:59.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:59.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:59.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:59.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:59.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:59.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:59.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:59.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:59.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:59.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:59.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:59.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:39:59.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:59.941 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 47235 47219 105.39 78.75 0.0147148 0.0094832 2023-10-09T06:40:00.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:00 smithi155 bash[59495]: cluster 2023-10-09T06:39:59.255273+0000 mgr.y (mgr.25723) 1060 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:40:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:00 smithi155 bash[59495]: cluster 2023-10-09T06:40:00.000076+0000 mon.a (mon.0) 1306 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:00 smithi155 bash[59495]: cluster 2023-10-09T06:40:00.000124+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:00.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:00.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:00.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.306 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[77299]: cluster 2023-10-09T06:39:59.255273+0000 mgr.y (mgr.25723) 1060 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:40:00.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[77299]: cluster 2023-10-09T06:40:00.000076+0000 mon.a (mon.0) 1306 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:00.307 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[77299]: cluster 2023-10-09T06:40:00.000124+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:00.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[80002]: cluster 2023-10-09T06:39:59.255273+0000 mgr.y (mgr.25723) 1060 : cluster 0 pgmap v1116: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:40:00.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[80002]: cluster 2023-10-09T06:40:00.000076+0000 mon.a (mon.0) 1306 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:00.307 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:00 smithi125 bash[80002]: cluster 2023-10-09T06:40:00.000124+0000 mon.a (mon.0) 1307 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:00.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:00.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:00.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:00.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:00.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:00.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:00.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:00.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:00.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 48443 48427 104.359 75.5 0.0205143 0.00957631 2023-10-09T06:40:00.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:00.956 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:40:00.956 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:40:00.956 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:40:00.956 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:40:00.956 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:40:00.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:00.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:01.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:01.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:01.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:01.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:01.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:01.281 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:40:01.295 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:40:01.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:01.360 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:01] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-10-09T06:40:01.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:01.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:01.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:01.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:01.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:01.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:01.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:01.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:01.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:01.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:01.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:01.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 49684 49668 103.465 77.5625 0.0126663 0.00965841 2023-10-09T06:40:01.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:02.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:02.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:02.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:02.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:02.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:02 smithi125 bash[77299]: cluster 2023-10-09T06:40:01.255917+0000 mgr.y (mgr.25723) 1061 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:40:02.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:02 smithi125 bash[77299]: audit 2023-10-09T06:40:01.260893+0000 mgr.y (mgr.25723) 1062 : audit 0 from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:02.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:02 smithi125 bash[80002]: cluster 2023-10-09T06:40:01.255917+0000 mgr.y (mgr.25723) 1061 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:40:02.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:02 smithi125 bash[80002]: audit 2023-10-09T06:40:01.260893+0000 mgr.y (mgr.25723) 1062 : audit 0 from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:02.411 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:02.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:02.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:02.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:02 smithi155 bash[59495]: cluster 2023-10-09T06:40:01.255917+0000 mgr.y (mgr.25723) 1061 : cluster 0 pgmap v1117: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:40:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:02 smithi155 bash[59495]: audit 2023-10-09T06:40:01.260893+0000 mgr.y (mgr.25723) 1062 : audit 0 from='client.55150 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:02.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:02.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:02.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:02.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:02.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:02.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:02.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 50967 50951 102.714 80.1875 0.0119203 0.00972923 2023-10-09T06:40:03.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:03.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:03.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:03.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:03.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:03.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:03.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:03.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:03.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:03.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:03.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:03.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:03.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:03.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:03.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 52210 52194 101.932 77.6875 0.0127522 0.00980453 2023-10-09T06:40:03.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:03.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:04.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:04.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:04.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:04.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:04.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:04.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:04.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:04.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:04 smithi125 bash[77299]: cluster 2023-10-09T06:40:03.256470+0000 mgr.y (mgr.25723) 1063 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:40:04.499 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:04 smithi125 bash[77299]: audit 2023-10-09T06:40:03.598700+0000 mon.a (mon.0) 1308 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:04.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:04 smithi125 bash[80002]: cluster 2023-10-09T06:40:03.256470+0000 mgr.y (mgr.25723) 1063 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:40:04.500 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:04 smithi125 bash[80002]: audit 2023-10-09T06:40:03.598700+0000 mon.a (mon.0) 1308 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:04.518 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:04.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:04.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:04 smithi155 bash[59495]: cluster 2023-10-09T06:40:03.256470+0000 mgr.y (mgr.25723) 1063 : cluster 0 pgmap v1118: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:40:04.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:04 smithi155 bash[59495]: audit 2023-10-09T06:40:03.598700+0000 mon.a (mon.0) 1308 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:04.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:04.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:04.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:04.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:04.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:04.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:04.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:04.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:04.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 53385 53369 101.068 73.4375 0.00983571 0.00988776 2023-10-09T06:40:05.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:05.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.189 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:05.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:05.378 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:40:05.424 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:40:05.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:05.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:05.478 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:40:05.530 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:40:05.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:05.567 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:40:05.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:05.602 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:40:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:05.646 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:40:05.682 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:40:05.690 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.690 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.691 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:40:05.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:05.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:05.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:05.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:05.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:05.942 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 54330 54314 99.8326 59.0625 0.00269608 0.00996659 2023-10-09T06:40:05.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:06.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:06.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:06.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:06.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:06.340 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:40:06.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:06.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:06.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:06.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:06.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:06.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[77299]: cluster 2023-10-09T06:40:05.257601+0000 mgr.y (mgr.25723) 1064 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T06:40:06.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[77299]: audit 2023-10-09T06:40:05.337966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.125:0/2228473560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:06.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[77299]: audit 2023-10-09T06:40:05.733020+0000 mon.a (mon.0) 1309 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:06.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[80002]: cluster 2023-10-09T06:40:05.257601+0000 mgr.y (mgr.25723) 1064 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T06:40:06.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[80002]: audit 2023-10-09T06:40:05.337966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.125:0/2228473560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:06.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:06 smithi125 bash[80002]: audit 2023-10-09T06:40:05.733020+0000 mon.a (mon.0) 1309 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:06 smithi155 bash[59495]: cluster 2023-10-09T06:40:05.257601+0000 mgr.y (mgr.25723) 1064 : cluster 0 pgmap v1119: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-10-09T06:40:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:06 smithi155 bash[59495]: audit 2023-10-09T06:40:05.337966+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.125:0/2228473560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:06 smithi155 bash[59495]: audit 2023-10-09T06:40:05.733020+0000 mon.a (mon.0) 1309 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:06.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:06.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:06.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:40:06.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:06.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 55314 55298 98.7354 61.5 0.0147554 0.0101203 2023-10-09T06:40:06.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:06.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:07.028 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1117,"stamp":"2023-10-09T06:40:01.255451+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2981369779,"num_objects":45447,"num_object_clones":0,"num_object_copies":136341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45447,"num_whiteouts":0,"num_read":6222,"num_read_kb":12658,"num_write":46525,"num_write_kb":2921975,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":2381407,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46589,"ondisk_log_size":46589,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12376964,"kb_used_data":8597900,"kb_used_omap":30,"kb_used_meta":3778977,"kb_avail":643839100,"statfs":{"total":671965249536,"available":659291238400,"internally_reserved":0,"allocated":8804249600,"data_stored":8798547857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":3869672559},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,5,6,5,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1172897792,"num_objects":17897,"num_object_clones":0,"num_object_copies":53691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17897,"num_write_kb":1145408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003382"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":228,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320081+0000","last_change":"2023-10-09T06:14:12.891368+0000","last_active":"2023-10-09T06:39:27.320081+0000","last_peered":"2023-10-09T06:39:27.320081+0000","last_clean":"2023-10-09T06:39:27.320081+0000","last_became_active":"2023-10-09T06:14:12.890317+0000","last_became_peered":"2023-10-09T06:14:12.890317+0000","last_unstale":"2023-10-09T06:39:27.320081+0000","last_undegraded":"2023-10-09T06:39:27.320081+0000","last_fullsized":"2023-10-09T06:39:27.320081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:30.009347+0000","last_change":"2023-10-09T06:39:21.064631+0000","last_active":"2023-10-09T06:39:30.009347+0000","last_peered":"2023-10-09T06:39:30.009347+0000","last_clean":"2023-10-09T06:39:30.009347+0000","last_became_active":"2023-10-09T06:39:21.064334+0000","last_became_peered":"2023-10-09T06:39:21.064334+0000","last_unstale":"2023-10-09T06:39:30.009347+0000","last_undegraded":"2023-10-09T06:39:30.009347+0000","last_fullsized":"2023-10-09T06:39:30.009347+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:53.742503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":96,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411599+0000","last_change":"2023-10-09T06:35:45.655708+0000","last_active":"2023-10-09T06:39:27.411599+0000","last_peered":"2023-10-09T06:39:27.411599+0000","last_clean":"2023-10-09T06:39:27.411599+0000","last_became_active":"2023-10-09T06:35:45.655323+0000","last_became_peered":"2023-10-09T06:35:45.655323+0000","last_unstale":"2023-10-09T06:39:27.411599+0000","last_undegraded":"2023-10-09T06:39:27.411599+0000","last_fullsized":"2023-10-09T06:39:27.411599+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:00.683127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":104,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418735+0000","last_change":"2023-10-09T06:38:49.383270+0000","last_active":"2023-10-09T06:39:27.418735+0000","last_peered":"2023-10-09T06:39:27.418735+0000","last_clean":"2023-10-09T06:39:27.418735+0000","last_became_active":"2023-10-09T06:38:49.382923+0000","last_became_peered":"2023-10-09T06:38:49.382923+0000","last_unstale":"2023-10-09T06:39:27.418735+0000","last_undegraded":"2023-10-09T06:39:27.418735+0000","last_fullsized":"2023-10-09T06:39:27.418735+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":241,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411197+0000","last_change":"2023-10-09T06:14:42.679929+0000","last_active":"2023-10-09T06:39:27.411197+0000","last_peered":"2023-10-09T06:39:27.411197+0000","last_clean":"2023-10-09T06:39:27.411197+0000","last_became_active":"2023-10-09T06:14:42.679580+0000","last_became_peered":"2023-10-09T06:14:42.679580+0000","last_unstale":"2023-10-09T06:39:27.411197+0000","last_undegraded":"2023-10-09T06:39:27.411197+0000","last_fullsized":"2023-10-09T06:39:27.411197+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:50.320788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417972+0000","last_change":"2023-10-09T06:38:22.271648+0000","last_active":"2023-10-09T06:39:27.417972+0000","last_peered":"2023-10-09T06:39:27.417972+0000","last_clean":"2023-10-09T06:39:27.417972+0000","last_became_active":"2023-10-09T06:38:22.271251+0000","last_became_peered":"2023-10-09T06:38:22.271251+0000","last_unstale":"2023-10-09T06:39:27.417972+0000","last_undegraded":"2023-10-09T06:39:27.417972+0000","last_fullsized":"2023-10-09T06:39:27.417972+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412335+0000","last_change":"2023-10-09T06:38:49.378992+0000","last_active":"2023-10-09T06:39:27.412335+0000","last_peered":"2023-10-09T06:39:27.412335+0000","last_clean":"2023-10-09T06:39:27.412335+0000","last_became_active":"2023-10-09T06:38:49.378664+0000","last_became_peered":"2023-10-09T06:38:49.378664+0000","last_unstale":"2023-10-09T06:39:27.412335+0000","last_undegraded":"2023-10-09T06:39:27.412335+0000","last_fullsized":"2023-10-09T06:39:27.412335+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:45.135324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411314+0000","last_change":"2023-10-09T06:28:31.017590+0000","last_active":"2023-10-09T06:39:27.411314+0000","last_peered":"2023-10-09T06:39:27.411314+0000","last_clean":"2023-10-09T06:39:27.411314+0000","last_became_active":"2023-10-09T06:28:31.017184+0000","last_became_peered":"2023-10-09T06:28:31.017184+0000","last_unstale":"2023-10-09T06:39:27.411314+0000","last_undegraded":"2023-10-09T06:39:27.411314+0000","last_fullsized":"2023-10-09T06:39:27.411314+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:13.465064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418998+0000","last_change":"2023-10-09T06:38:22.275387+0000","last_active":"2023-10-09T06:39:27.418998+0000","last_peered":"2023-10-09T06:39:27.418998+0000","last_clean":"2023-10-09T06:39:27.418998+0000","last_became_active":"2023-10-09T06:38:22.275008+0000","last_became_peered":"2023-10-09T06:38:22.275008+0000","last_unstale":"2023-10-09T06:39:27.418998+0000","last_undegraded":"2023-10-09T06:39:27.418998+0000","last_fullsized":"2023-10-09T06:39:27.418998+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":251,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411537+0000","last_change":"2023-10-09T06:28:31.016581+0000","last_active":"2023-10-09T06:39:27.411537+0000","last_peered":"2023-10-09T06:39:27.411537+0000","last_clean":"2023-10-09T06:39:27.411537+0000","last_became_active":"2023-10-09T06:28:31.016177+0000","last_became_peered":"2023-10-09T06:28:31.016177+0000","last_unstale":"2023-10-09T06:39:27.411537+0000","last_undegraded":"2023-10-09T06:39:27.411537+0000","last_fullsized":"2023-10-09T06:39:27.411537+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:49.521920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":145,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.419009+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:39:27.419009+0000","last_peered":"2023-10-09T06:39:27.419009+0000","last_clean":"2023-10-09T06:39:27.419009+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:39:27.419009+0000","last_undegraded":"2023-10-09T06:39:27.419009+0000","last_fullsized":"2023-10-09T06:39:27.419009+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":186,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417133+0000","last_change":"2023-10-09T06:35:45.663547+0000","last_active":"2023-10-09T06:39:27.417133+0000","last_peered":"2023-10-09T06:39:27.417133+0000","last_clean":"2023-10-09T06:39:27.417133+0000","last_became_active":"2023-10-09T06:35:45.663179+0000","last_became_peered":"2023-10-09T06:35:45.663179+0000","last_unstale":"2023-10-09T06:39:27.417133+0000","last_undegraded":"2023-10-09T06:39:27.417133+0000","last_fullsized":"2023-10-09T06:39:27.417133+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":295,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320485+0000","last_change":"2023-10-09T06:28:31.039897+0000","last_active":"2023-10-09T06:39:27.320485+0000","last_peered":"2023-10-09T06:39:27.320485+0000","last_clean":"2023-10-09T06:39:27.320485+0000","last_became_active":"2023-10-09T06:28:31.027533+0000","last_became_peered":"2023-10-09T06:28:31.027533+0000","last_unstale":"2023-10-09T06:39:27.320485+0000","last_undegraded":"2023-10-09T06:39:27.320485+0000","last_fullsized":"2023-10-09T06:39:27.320485+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320365+0000","last_change":"2023-10-09T06:28:31.029048+0000","last_active":"2023-10-09T06:39:27.320365+0000","last_peered":"2023-10-09T06:39:27.320365+0000","last_clean":"2023-10-09T06:39:27.320365+0000","last_became_active":"2023-10-09T06:28:31.028203+0000","last_became_peered":"2023-10-09T06:28:31.028203+0000","last_unstale":"2023-10-09T06:39:27.320365+0000","last_undegraded":"2023-10-09T06:39:27.320365+0000","last_fullsized":"2023-10-09T06:39:27.320365+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":145,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418305+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:39:27.418305+0000","last_peered":"2023-10-09T06:39:27.418305+0000","last_clean":"2023-10-09T06:39:27.418305+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:39:27.418305+0000","last_undegraded":"2023-10-09T06:39:27.418305+0000","last_fullsized":"2023-10-09T06:39:27.418305+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411791+0000","last_change":"2023-10-09T06:18:59.793166+0000","last_active":"2023-10-09T06:39:27.411791+0000","last_peered":"2023-10-09T06:39:27.411791+0000","last_clean":"2023-10-09T06:39:27.411791+0000","last_became_active":"2023-10-09T06:18:59.792826+0000","last_became_peered":"2023-10-09T06:18:59.792826+0000","last_unstale":"2023-10-09T06:39:27.411791+0000","last_undegraded":"2023-10-09T06:39:27.411791+0000","last_fullsized":"2023-10-09T06:39:27.411791+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:58.295241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417321+0000","last_change":"2023-10-09T06:38:49.382202+0000","last_active":"2023-10-09T06:39:27.417321+0000","last_peered":"2023-10-09T06:39:27.417321+0000","last_clean":"2023-10-09T06:39:27.417321+0000","last_became_active":"2023-10-09T06:38:49.381744+0000","last_became_peered":"2023-10-09T06:38:49.381744+0000","last_unstale":"2023-10-09T06:39:27.417321+0000","last_undegraded":"2023-10-09T06:39:27.417321+0000","last_fullsized":"2023-10-09T06:39:27.417321+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418864+0000","last_change":"2023-10-09T06:38:22.283308+0000","last_active":"2023-10-09T06:39:27.418864+0000","last_peered":"2023-10-09T06:39:27.418864+0000","last_clean":"2023-10-09T06:39:27.418864+0000","last_became_active":"2023-10-09T06:38:22.282575+0000","last_became_peered":"2023-10-09T06:38:22.282575+0000","last_unstale":"2023-10-09T06:39:27.418864+0000","last_undegraded":"2023-10-09T06:39:27.418864+0000","last_fullsized":"2023-10-09T06:39:27.418864+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":166,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418680+0000","last_change":"2023-10-09T06:39:21.028232+0000","last_active":"2023-10-09T06:39:27.418680+0000","last_peered":"2023-10-09T06:39:27.418680+0000","last_clean":"2023-10-09T06:39:27.418680+0000","last_became_active":"2023-10-09T06:39:21.027741+0000","last_became_peered":"2023-10-09T06:39:21.027741+0000","last_unstale":"2023-10-09T06:39:27.418680+0000","last_undegraded":"2023-10-09T06:39:27.418680+0000","last_fullsized":"2023-10-09T06:39:27.418680+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":95,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412334+0000","last_change":"2023-10-09T06:38:49.378852+0000","last_active":"2023-10-09T06:39:27.412334+0000","last_peered":"2023-10-09T06:39:27.412334+0000","last_clean":"2023-10-09T06:39:27.412334+0000","last_became_active":"2023-10-09T06:38:49.378523+0000","last_became_peered":"2023-10-09T06:38:49.378523+0000","last_unstale":"2023-10-09T06:39:27.412334+0000","last_undegraded":"2023-10-09T06:39:27.412334+0000","last_fullsized":"2023-10-09T06:39:27.412334+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:42.477954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":116,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.416423+0000","last_change":"2023-10-09T06:38:22.538210+0000","last_active":"2023-10-09T06:39:27.416423+0000","last_peered":"2023-10-09T06:39:27.416423+0000","last_clean":"2023-10-09T06:39:27.416423+0000","last_became_active":"2023-10-09T06:38:22.283807+0000","last_became_peered":"2023-10-09T06:38:22.283807+0000","last_unstale":"2023-10-09T06:39:27.416423+0000","last_undegraded":"2023-10-09T06:39:27.416423+0000","last_fullsized":"2023-10-09T06:39:27.416423+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":256,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412647+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:39:27.412647+0000","last_peered":"2023-10-09T06:39:27.412647+0000","last_clean":"2023-10-09T06:39:27.412647+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:39:27.412647+0000","last_undegraded":"2023-10-09T06:39:27.412647+0000","last_fullsized":"2023-10-09T06:39:27.412647+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"247'2912","reported_seq":2925,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.454378+0000","last_change":"2023-10-09T06:39:06.462564+0000","last_active":"2023-10-09T06:39:59.454378+0000","last_peered":"2023-10-09T06:39:59.454378+0000","last_clean":"2023-10-09T06:39:59.454378+0000","last_became_active":"2023-10-09T06:39:06.461749+0000","last_became_peered":"2023-10-09T06:39:06.461749+0000","last_unstale":"2023-10-09T06:39:59.454378+0000","last_undegraded":"2023-10-09T06:39:59.454378+0000","last_fullsized":"2023-10-09T06:39:59.454378+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2912,"log_dups_size":0,"ondisk_log_size":2912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2912,"num_write_kb":186368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":143,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.419358+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:39:27.419358+0000","last_peered":"2023-10-09T06:39:27.419358+0000","last_clean":"2023-10-09T06:39:27.419358+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:39:27.419358+0000","last_undegraded":"2023-10-09T06:39:27.419358+0000","last_fullsized":"2023-10-09T06:39:27.419358+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417624+0000","last_change":"2023-10-09T06:38:22.275464+0000","last_active":"2023-10-09T06:39:27.417624+0000","last_peered":"2023-10-09T06:39:27.417624+0000","last_clean":"2023-10-09T06:39:27.417624+0000","last_became_active":"2023-10-09T06:38:22.275023+0000","last_became_peered":"2023-10-09T06:38:22.275023+0000","last_unstale":"2023-10-09T06:39:27.417624+0000","last_undegraded":"2023-10-09T06:39:27.417624+0000","last_fullsized":"2023-10-09T06:39:27.417624+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"247'2948","reported_seq":2961,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.431974+0000","last_change":"2023-10-09T06:39:06.457503+0000","last_active":"2023-10-09T06:39:59.431974+0000","last_peered":"2023-10-09T06:39:59.431974+0000","last_clean":"2023-10-09T06:39:59.431974+0000","last_became_active":"2023-10-09T06:39:06.456829+0000","last_became_peered":"2023-10-09T06:39:06.456829+0000","last_unstale":"2023-10-09T06:39:59.431974+0000","last_undegraded":"2023-10-09T06:39:59.431974+0000","last_fullsized":"2023-10-09T06:39:59.431974+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2948,"log_dups_size":0,"ondisk_log_size":2948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193200128,"num_objects":2948,"num_object_clones":0,"num_object_copies":8844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2948,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":143,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.419229+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:39:27.419229+0000","last_peered":"2023-10-09T06:39:27.419229+0000","last_clean":"2023-10-09T06:39:27.419229+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:39:27.419229+0000","last_undegraded":"2023-10-09T06:39:27.419229+0000","last_fullsized":"2023-10-09T06:39:27.419229+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":226,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320089+0000","last_change":"2023-10-09T06:14:42.688520+0000","last_active":"2023-10-09T06:39:27.320089+0000","last_peered":"2023-10-09T06:39:27.320089+0000","last_clean":"2023-10-09T06:39:27.320089+0000","last_became_active":"2023-10-09T06:14:42.688146+0000","last_became_peered":"2023-10-09T06:14:42.688146+0000","last_unstale":"2023-10-09T06:39:27.320089+0000","last_undegraded":"2023-10-09T06:39:27.320089+0000","last_fullsized":"2023-10-09T06:39:27.320089+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"247'2922","reported_seq":2936,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.423142+0000","last_change":"2023-10-09T06:39:06.335034+0000","last_active":"2023-10-09T06:39:59.423142+0000","last_peered":"2023-10-09T06:39:59.423142+0000","last_clean":"2023-10-09T06:39:59.423142+0000","last_became_active":"2023-10-09T06:39:06.334779+0000","last_became_peered":"2023-10-09T06:39:06.334779+0000","last_unstale":"2023-10-09T06:39:59.423142+0000","last_undegraded":"2023-10-09T06:39:59.423142+0000","last_fullsized":"2023-10-09T06:39:59.423142+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2922,"log_dups_size":0,"ondisk_log_size":2922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191496192,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2922,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":182,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418175+0000","last_change":"2023-10-09T06:38:22.365385+0000","last_active":"2023-10-09T06:39:27.418175+0000","last_peered":"2023-10-09T06:39:27.418175+0000","last_clean":"2023-10-09T06:39:27.418175+0000","last_became_active":"2023-10-09T06:38:22.277152+0000","last_became_peered":"2023-10-09T06:38:22.277152+0000","last_unstale":"2023-10-09T06:39:27.418175+0000","last_undegraded":"2023-10-09T06:39:27.418175+0000","last_fullsized":"2023-10-09T06:39:27.418175+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.416956+0000","last_change":"2023-10-09T06:28:31.035817+0000","last_active":"2023-10-09T06:39:27.416956+0000","last_peered":"2023-10-09T06:39:27.416956+0000","last_clean":"2023-10-09T06:39:27.416956+0000","last_became_active":"2023-10-09T06:28:31.035425+0000","last_became_peered":"2023-10-09T06:28:31.035425+0000","last_unstale":"2023-10-09T06:39:27.416956+0000","last_undegraded":"2023-10-09T06:39:27.416956+0000","last_fullsized":"2023-10-09T06:39:27.416956+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"247'2553","reported_seq":2568,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:56.501414+0000","last_change":"2023-10-09T06:39:06.453038+0000","last_active":"2023-10-09T06:39:56.501414+0000","last_peered":"2023-10-09T06:39:56.501414+0000","last_clean":"2023-10-09T06:39:56.501414+0000","last_became_active":"2023-10-09T06:39:06.452620+0000","last_became_peered":"2023-10-09T06:39:06.452620+0000","last_unstale":"2023-10-09T06:39:56.501414+0000","last_undegraded":"2023-10-09T06:39:56.501414+0000","last_fullsized":"2023-10-09T06:39:56.501414+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2553,"log_dups_size":0,"ondisk_log_size":2553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:58.935766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2553,"num_write_kb":163392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411998+0000","last_change":"2023-10-09T06:38:22.267196+0000","last_active":"2023-10-09T06:39:27.411998+0000","last_peered":"2023-10-09T06:39:27.411998+0000","last_clean":"2023-10-09T06:39:27.411998+0000","last_became_active":"2023-10-09T06:38:22.266748+0000","last_became_peered":"2023-10-09T06:38:22.266748+0000","last_unstale":"2023-10-09T06:39:27.411998+0000","last_undegraded":"2023-10-09T06:39:27.411998+0000","last_fullsized":"2023-10-09T06:39:27.411998+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:28.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":205,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311670+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:39:27.311670+0000","last_peered":"2023-10-09T06:39:27.311670+0000","last_clean":"2023-10-09T06:39:27.311670+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:39:27.311670+0000","last_undegraded":"2023-10-09T06:39:27.311670+0000","last_fullsized":"2023-10-09T06:39:27.311670+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"247'2774","reported_seq":2789,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:57.464796+0000","last_change":"2023-10-09T06:39:06.322576+0000","last_active":"2023-10-09T06:39:57.464796+0000","last_peered":"2023-10-09T06:39:57.464796+0000","last_clean":"2023-10-09T06:39:57.464796+0000","last_became_active":"2023-10-09T06:39:06.322247+0000","last_became_peered":"2023-10-09T06:39:06.322247+0000","last_unstale":"2023-10-09T06:39:57.464796+0000","last_undegraded":"2023-10-09T06:39:57.464796+0000","last_fullsized":"2023-10-09T06:39:57.464796+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2774,"log_dups_size":0,"ondisk_log_size":2774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:37.101909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181796864,"num_objects":2774,"num_object_clones":0,"num_object_copies":8322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2774,"num_write_kb":177536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":225,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.433260+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:39:27.433260+0000","last_peered":"2023-10-09T06:39:27.433260+0000","last_clean":"2023-10-09T06:39:27.433260+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:39:27.433260+0000","last_undegraded":"2023-10-09T06:39:27.433260+0000","last_fullsized":"2023-10-09T06:39:27.433260+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320438+0000","last_change":"2023-10-09T06:28:31.029851+0000","last_active":"2023-10-09T06:39:27.320438+0000","last_peered":"2023-10-09T06:39:27.320438+0000","last_clean":"2023-10-09T06:39:27.320438+0000","last_became_active":"2023-10-09T06:28:31.028525+0000","last_became_peered":"2023-10-09T06:28:31.028525+0000","last_unstale":"2023-10-09T06:39:27.320438+0000","last_undegraded":"2023-10-09T06:39:27.320438+0000","last_fullsized":"2023-10-09T06:39:27.320438+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"247'2675","reported_seq":2690,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:56.495705+0000","last_change":"2023-10-09T06:39:06.452410+0000","last_active":"2023-10-09T06:39:56.495705+0000","last_peered":"2023-10-09T06:39:56.495705+0000","last_clean":"2023-10-09T06:39:56.495705+0000","last_became_active":"2023-10-09T06:39:06.451876+0000","last_became_peered":"2023-10-09T06:39:06.451876+0000","last_unstale":"2023-10-09T06:39:56.495705+0000","last_undegraded":"2023-10-09T06:39:56.495705+0000","last_fullsized":"2023-10-09T06:39:56.495705+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2675,"log_dups_size":0,"ondisk_log_size":2675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:39.352667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2675,"num_write_kb":171200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311524+0000","last_change":"2023-10-09T06:39:21.344045+0000","last_active":"2023-10-09T06:39:27.311524+0000","last_peered":"2023-10-09T06:39:27.311524+0000","last_clean":"2023-10-09T06:39:27.311524+0000","last_became_active":"2023-10-09T06:39:21.343604+0000","last_became_peered":"2023-10-09T06:39:21.343604+0000","last_unstale":"2023-10-09T06:39:27.311524+0000","last_undegraded":"2023-10-09T06:39:27.311524+0000","last_fullsized":"2023-10-09T06:39:27.311524+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:54.025536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":226,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320595+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:39:27.320595+0000","last_peered":"2023-10-09T06:39:27.320595+0000","last_clean":"2023-10-09T06:39:27.320595+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:39:27.320595+0000","last_undegraded":"2023-10-09T06:39:27.320595+0000","last_fullsized":"2023-10-09T06:39:27.320595+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"247'2953","reported_seq":2966,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.447619+0000","last_change":"2023-10-09T06:39:06.459420+0000","last_active":"2023-10-09T06:39:59.447619+0000","last_peered":"2023-10-09T06:39:59.447619+0000","last_clean":"2023-10-09T06:39:59.447619+0000","last_became_active":"2023-10-09T06:39:06.458646+0000","last_became_peered":"2023-10-09T06:39:06.458646+0000","last_unstale":"2023-10-09T06:39:59.447619+0000","last_undegraded":"2023-10-09T06:39:59.447619+0000","last_fullsized":"2023-10-09T06:39:59.447619+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2953,"log_dups_size":0,"ondisk_log_size":2953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193527808,"num_objects":2953,"num_object_clones":0,"num_object_copies":8859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2953,"num_write_kb":188992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.319782+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:39:27.319782+0000","last_peered":"2023-10-09T06:39:27.319782+0000","last_clean":"2023-10-09T06:39:27.319782+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:39:27.319782+0000","last_undegraded":"2023-10-09T06:39:27.319782+0000","last_fullsized":"2023-10-09T06:39:27.319782+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320848+0000","last_change":"2023-10-09T06:28:31.029609+0000","last_active":"2023-10-09T06:39:27.320848+0000","last_peered":"2023-10-09T06:39:27.320848+0000","last_clean":"2023-10-09T06:39:27.320848+0000","last_became_active":"2023-10-09T06:28:31.027199+0000","last_became_peered":"2023-10-09T06:28:31.027199+0000","last_unstale":"2023-10-09T06:39:27.320848+0000","last_undegraded":"2023-10-09T06:39:27.320848+0000","last_fullsized":"2023-10-09T06:39:27.320848+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"247'2765","reported_seq":2779,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:57.476166+0000","last_change":"2023-10-09T06:39:06.454044+0000","last_active":"2023-10-09T06:39:57.476166+0000","last_peered":"2023-10-09T06:39:57.476166+0000","last_clean":"2023-10-09T06:39:57.476166+0000","last_became_active":"2023-10-09T06:39:06.453738+0000","last_became_peered":"2023-10-09T06:39:06.453738+0000","last_unstale":"2023-10-09T06:39:57.476166+0000","last_undegraded":"2023-10-09T06:39:57.476166+0000","last_fullsized":"2023-10-09T06:39:57.476166+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2765,"log_dups_size":0,"ondisk_log_size":2765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:47:26.269570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2765,"num_write_kb":176960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":245,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411418+0000","last_change":"2023-10-09T06:18:59.799345+0000","last_active":"2023-10-09T06:39:27.411418+0000","last_peered":"2023-10-09T06:39:27.411418+0000","last_clean":"2023-10-09T06:39:27.411418+0000","last_became_active":"2023-10-09T06:18:59.798917+0000","last_became_peered":"2023-10-09T06:18:59.798917+0000","last_unstale":"2023-10-09T06:39:27.411418+0000","last_undegraded":"2023-10-09T06:39:27.411418+0000","last_fullsized":"2023-10-09T06:39:27.411418+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:52.400840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":97,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412492+0000","last_change":"2023-10-09T06:35:45.679016+0000","last_active":"2023-10-09T06:39:27.412492+0000","last_peered":"2023-10-09T06:39:27.412492+0000","last_clean":"2023-10-09T06:39:27.412492+0000","last_became_active":"2023-10-09T06:35:45.678002+0000","last_became_peered":"2023-10-09T06:35:45.678002+0000","last_unstale":"2023-10-09T06:39:27.412492+0000","last_undegraded":"2023-10-09T06:39:27.412492+0000","last_fullsized":"2023-10-09T06:39:27.412492+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:22.262853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"247'2679","reported_seq":2694,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:57.465806+0000","last_change":"2023-10-09T06:39:06.451992+0000","last_active":"2023-10-09T06:39:57.465806+0000","last_peered":"2023-10-09T06:39:57.465806+0000","last_clean":"2023-10-09T06:39:57.465806+0000","last_became_active":"2023-10-09T06:39:06.451648+0000","last_became_peered":"2023-10-09T06:39:06.451648+0000","last_unstale":"2023-10-09T06:39:57.465806+0000","last_undegraded":"2023-10-09T06:39:57.465806+0000","last_fullsized":"2023-10-09T06:39:57.465806+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2679,"log_dups_size":0,"ondisk_log_size":2679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:39.385944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175570944,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2679,"num_write_kb":171456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":940,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417502+0000","last_change":"2023-10-09T06:38:49.382814+0000","last_active":"2023-10-09T06:39:27.417502+0000","last_peered":"2023-10-09T06:39:27.417502+0000","last_clean":"2023-10-09T06:39:27.417502+0000","last_became_active":"2023-10-09T06:38:49.382305+0000","last_became_peered":"2023-10-09T06:38:49.382305+0000","last_unstale":"2023-10-09T06:39:27.417502+0000","last_undegraded":"2023-10-09T06:39:27.417502+0000","last_fullsized":"2023-10-09T06:39:27.417502+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":101,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411919+0000","last_change":"2023-10-09T06:38:22.270825+0000","last_active":"2023-10-09T06:39:27.411919+0000","last_peered":"2023-10-09T06:39:27.411919+0000","last_clean":"2023-10-09T06:39:27.411919+0000","last_became_active":"2023-10-09T06:38:22.270452+0000","last_became_peered":"2023-10-09T06:38:22.270452+0000","last_unstale":"2023-10-09T06:39:27.411919+0000","last_undegraded":"2023-10-09T06:39:27.411919+0000","last_fullsized":"2023-10-09T06:39:27.411919+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:32.439859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":43,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.319969+0000","last_change":"2023-10-09T06:38:22.274870+0000","last_active":"2023-10-09T06:39:27.319969+0000","last_peered":"2023-10-09T06:39:27.319969+0000","last_clean":"2023-10-09T06:39:27.319969+0000","last_became_active":"2023-10-09T06:38:22.273614+0000","last_became_peered":"2023-10-09T06:38:22.273614+0000","last_unstale":"2023-10-09T06:39:27.319969+0000","last_undegraded":"2023-10-09T06:39:27.319969+0000","last_fullsized":"2023-10-09T06:39:27.319969+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"247'2977","reported_seq":2991,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.440513+0000","last_change":"2023-10-09T06:39:06.331577+0000","last_active":"2023-10-09T06:39:59.440513+0000","last_peered":"2023-10-09T06:39:59.440513+0000","last_clean":"2023-10-09T06:39:59.440513+0000","last_became_active":"2023-10-09T06:39:06.330981+0000","last_became_peered":"2023-10-09T06:39:06.330981+0000","last_unstale":"2023-10-09T06:39:59.440513+0000","last_undegraded":"2023-10-09T06:39:59.440513+0000","last_fullsized":"2023-10-09T06:39:59.440513+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2977,"log_dups_size":0,"ondisk_log_size":2977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":43,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311531+0000","last_change":"2023-10-09T06:38:22.378334+0000","last_active":"2023-10-09T06:39:27.311531+0000","last_peered":"2023-10-09T06:39:27.311531+0000","last_clean":"2023-10-09T06:39:27.311531+0000","last_became_active":"2023-10-09T06:38:22.268209+0000","last_became_peered":"2023-10-09T06:38:22.268209+0000","last_unstale":"2023-10-09T06:39:27.311531+0000","last_undegraded":"2023-10-09T06:39:27.311531+0000","last_fullsized":"2023-10-09T06:39:27.311531+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:55:27.555221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":883,"num_read_kb":1177,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320219+0000","last_change":"2023-10-09T06:28:31.028131+0000","last_active":"2023-10-09T06:39:27.320219+0000","last_peered":"2023-10-09T06:39:27.320219+0000","last_clean":"2023-10-09T06:39:27.320219+0000","last_became_active":"2023-10-09T06:28:31.027793+0000","last_became_peered":"2023-10-09T06:28:31.027793+0000","last_unstale":"2023-10-09T06:39:27.320219+0000","last_undegraded":"2023-10-09T06:39:27.320219+0000","last_fullsized":"2023-10-09T06:39:27.320219+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":113,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:30.234983+0000","last_change":"2023-10-09T06:35:45.655403+0000","last_active":"2023-10-09T06:39:30.234983+0000","last_peered":"2023-10-09T06:39:30.234983+0000","last_clean":"2023-10-09T06:39:30.234983+0000","last_became_active":"2023-10-09T06:35:45.655106+0000","last_became_peered":"2023-10-09T06:35:45.655106+0000","last_unstale":"2023-10-09T06:39:30.234983+0000","last_undegraded":"2023-10-09T06:39:30.234983+0000","last_fullsized":"2023-10-09T06:39:30.234983+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:10:20.470509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"247'2942","reported_seq":2956,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.420556+0000","last_change":"2023-10-09T06:39:06.459488+0000","last_active":"2023-10-09T06:39:59.420556+0000","last_peered":"2023-10-09T06:39:59.420556+0000","last_clean":"2023-10-09T06:39:59.420556+0000","last_became_active":"2023-10-09T06:39:06.458713+0000","last_became_peered":"2023-10-09T06:39:06.458713+0000","last_unstale":"2023-10-09T06:39:59.420556+0000","last_undegraded":"2023-10-09T06:39:59.420556+0000","last_fullsized":"2023-10-09T06:39:59.420556+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2942,"log_dups_size":0,"ondisk_log_size":2942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2942,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":614,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411077+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:39:27.411077+0000","last_peered":"2023-10-09T06:39:27.411077+0000","last_clean":"2023-10-09T06:39:27.411077+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:39:27.411077+0000","last_undegraded":"2023-10-09T06:39:27.411077+0000","last_fullsized":"2023-10-09T06:39:27.411077+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":188,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:30.234794+0000","last_change":"2023-10-09T06:35:45.655401+0000","last_active":"2023-10-09T06:39:30.234794+0000","last_peered":"2023-10-09T06:39:30.234794+0000","last_clean":"2023-10-09T06:39:30.234794+0000","last_became_active":"2023-10-09T06:35:45.654688+0000","last_became_peered":"2023-10-09T06:35:45.654688+0000","last_unstale":"2023-10-09T06:39:30.234794+0000","last_undegraded":"2023-10-09T06:39:30.234794+0000","last_fullsized":"2023-10-09T06:39:30.234794+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:17.726367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418954+0000","last_change":"2023-10-09T06:38:49.381966+0000","last_active":"2023-10-09T06:39:27.418954+0000","last_peered":"2023-10-09T06:39:27.418954+0000","last_clean":"2023-10-09T06:39:27.418954+0000","last_became_active":"2023-10-09T06:38:49.381559+0000","last_became_peered":"2023-10-09T06:38:49.381559+0000","last_unstale":"2023-10-09T06:39:27.418954+0000","last_undegraded":"2023-10-09T06:39:27.418954+0000","last_fullsized":"2023-10-09T06:39:27.418954+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.8","version":"247'2780","reported_seq":2795,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:57.474974+0000","last_change":"2023-10-09T06:39:06.322429+0000","last_active":"2023-10-09T06:39:57.474974+0000","last_peered":"2023-10-09T06:39:57.474974+0000","last_clean":"2023-10-09T06:39:57.474974+0000","last_became_active":"2023-10-09T06:39:06.322072+0000","last_became_peered":"2023-10-09T06:39:06.322072+0000","last_unstale":"2023-10-09T06:39:57.474974+0000","last_undegraded":"2023-10-09T06:39:57.474974+0000","last_fullsized":"2023-10-09T06:39:57.474974+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2780,"log_dups_size":0,"ondisk_log_size":2780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:43.217621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182190080,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":177920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":384,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412016+0000","last_change":"2023-10-09T06:38:25.001852+0000","last_active":"2023-10-09T06:39:27.412016+0000","last_peered":"2023-10-09T06:39:27.412016+0000","last_clean":"2023-10-09T06:39:27.412016+0000","last_became_active":"2023-10-09T06:38:25.001121+0000","last_became_peered":"2023-10-09T06:38:25.001121+0000","last_unstale":"2023-10-09T06:39:27.412016+0000","last_undegraded":"2023-10-09T06:39:27.412016+0000","last_fullsized":"2023-10-09T06:39:27.412016+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:16.241721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":307,"num_read_kb":1996,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":489,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.416774+0000","last_change":"2023-10-09T06:28:34.038449+0000","last_active":"2023-10-09T06:39:27.416774+0000","last_peered":"2023-10-09T06:39:27.416774+0000","last_clean":"2023-10-09T06:39:27.416774+0000","last_became_active":"2023-10-09T06:28:34.037788+0000","last_became_peered":"2023-10-09T06:28:34.037788+0000","last_unstale":"2023-10-09T06:39:27.416774+0000","last_undegraded":"2023-10-09T06:39:27.416774+0000","last_fullsized":"2023-10-09T06:39:27.416774+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:30.008851+0000","last_change":"2023-10-09T06:38:22.267011+0000","last_active":"2023-10-09T06:39:30.008851+0000","last_peered":"2023-10-09T06:39:30.008851+0000","last_clean":"2023-10-09T06:39:30.008851+0000","last_became_active":"2023-10-09T06:38:22.266628+0000","last_became_peered":"2023-10-09T06:38:22.266628+0000","last_unstale":"2023-10-09T06:39:30.008851+0000","last_undegraded":"2023-10-09T06:39:30.008851+0000","last_fullsized":"2023-10-09T06:39:30.008851+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:07.568040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":202,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320770+0000","last_change":"2023-10-09T06:28:31.027561+0000","last_active":"2023-10-09T06:39:27.320770+0000","last_peered":"2023-10-09T06:39:27.320770+0000","last_clean":"2023-10-09T06:39:27.320770+0000","last_became_active":"2023-10-09T06:28:31.027190+0000","last_became_peered":"2023-10-09T06:28:31.027190+0000","last_unstale":"2023-10-09T06:39:27.320770+0000","last_undegraded":"2023-10-09T06:39:27.320770+0000","last_fullsized":"2023-10-09T06:39:27.320770+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"247'2827","reported_seq":2842,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.335645+0000","last_change":"2023-10-09T06:39:06.324796+0000","last_active":"2023-10-09T06:39:59.335645+0000","last_peered":"2023-10-09T06:39:59.335645+0000","last_clean":"2023-10-09T06:39:59.335645+0000","last_became_active":"2023-10-09T06:39:06.324156+0000","last_became_peered":"2023-10-09T06:39:06.324156+0000","last_unstale":"2023-10-09T06:39:59.335645+0000","last_undegraded":"2023-10-09T06:39:59.335645+0000","last_fullsized":"2023-10-09T06:39:59.335645+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2827,"log_dups_size":0,"ondisk_log_size":2827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:07.479555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185270272,"num_objects":2827,"num_object_clones":0,"num_object_copies":8481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2827,"num_write_kb":180928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":168,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.417368+0000","last_change":"2023-10-09T06:19:01.161539+0000","last_active":"2023-10-09T06:39:27.417368+0000","last_peered":"2023-10-09T06:39:27.417368+0000","last_clean":"2023-10-09T06:39:27.417368+0000","last_became_active":"2023-10-09T06:18:59.805328+0000","last_became_peered":"2023-10-09T06:18:59.805328+0000","last_unstale":"2023-10-09T06:39:27.417368+0000","last_undegraded":"2023-10-09T06:39:27.417368+0000","last_fullsized":"2023-10-09T06:39:27.417368+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":206,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311523+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:39:27.311523+0000","last_peered":"2023-10-09T06:39:27.311523+0000","last_clean":"2023-10-09T06:39:27.311523+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:39:27.311523+0000","last_undegraded":"2023-10-09T06:39:27.311523+0000","last_fullsized":"2023-10-09T06:39:27.311523+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":221,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311956+0000","last_change":"2023-10-09T06:14:12.887525+0000","last_active":"2023-10-09T06:39:27.311956+0000","last_peered":"2023-10-09T06:39:27.311956+0000","last_clean":"2023-10-09T06:39:27.311956+0000","last_became_active":"2023-10-09T06:14:12.887165+0000","last_became_peered":"2023-10-09T06:14:12.887165+0000","last_unstale":"2023-10-09T06:39:27.311956+0000","last_undegraded":"2023-10-09T06:39:27.311956+0000","last_fullsized":"2023-10-09T06:39:27.311956+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:09.852049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"247'2893","reported_seq":2907,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.455306+0000","last_change":"2023-10-09T06:39:06.458845+0000","last_active":"2023-10-09T06:39:59.455306+0000","last_peered":"2023-10-09T06:39:59.455306+0000","last_clean":"2023-10-09T06:39:59.455306+0000","last_became_active":"2023-10-09T06:39:06.458327+0000","last_became_peered":"2023-10-09T06:39:06.458327+0000","last_unstale":"2023-10-09T06:39:59.455306+0000","last_undegraded":"2023-10-09T06:39:59.455306+0000","last_fullsized":"2023-10-09T06:39:59.455306+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2893,"log_dups_size":0,"ondisk_log_size":2893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189595648,"num_objects":2893,"num_object_clones":0,"num_object_copies":8679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2893,"num_write_kb":185152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":157,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418389+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:39:27.418389+0000","last_peered":"2023-10-09T06:39:27.418389+0000","last_clean":"2023-10-09T06:39:27.418389+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:39:27.418389+0000","last_undegraded":"2023-10-09T06:39:27.418389+0000","last_fullsized":"2023-10-09T06:39:27.418389+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":101,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:30.235165+0000","last_change":"2023-10-09T06:38:22.271659+0000","last_active":"2023-10-09T06:39:30.235165+0000","last_peered":"2023-10-09T06:39:30.235165+0000","last_clean":"2023-10-09T06:39:30.235165+0000","last_became_active":"2023-10-09T06:38:22.271318+0000","last_became_peered":"2023-10-09T06:38:22.271318+0000","last_unstale":"2023-10-09T06:39:30.235165+0000","last_undegraded":"2023-10-09T06:39:30.235165+0000","last_fullsized":"2023-10-09T06:39:30.235165+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:27.271581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418145+0000","last_change":"2023-10-09T06:38:49.381891+0000","last_active":"2023-10-09T06:39:27.418145+0000","last_peered":"2023-10-09T06:39:27.418145+0000","last_clean":"2023-10-09T06:39:27.418145+0000","last_became_active":"2023-10-09T06:38:49.381442+0000","last_became_peered":"2023-10-09T06:38:49.381442+0000","last_unstale":"2023-10-09T06:39:27.418145+0000","last_undegraded":"2023-10-09T06:39:27.418145+0000","last_fullsized":"2023-10-09T06:39:27.418145+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"247'2972","reported_seq":2986,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:59.440679+0000","last_change":"2023-10-09T06:39:06.459859+0000","last_active":"2023-10-09T06:39:59.440679+0000","last_peered":"2023-10-09T06:39:59.440679+0000","last_clean":"2023-10-09T06:39:59.440679+0000","last_became_active":"2023-10-09T06:39:06.459222+0000","last_became_peered":"2023-10-09T06:39:06.459222+0000","last_unstale":"2023-10-09T06:39:59.440679+0000","last_undegraded":"2023-10-09T06:39:59.440679+0000","last_fullsized":"2023-10-09T06:39:59.440679+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2972,"log_dups_size":0,"ondisk_log_size":2972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194772992,"num_objects":2972,"num_object_clones":0,"num_object_copies":8916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2972,"num_write_kb":190208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":251,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412523+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:39:27.412523+0000","last_peered":"2023-10-09T06:39:27.412523+0000","last_clean":"2023-10-09T06:39:27.412523+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:39:27.412523+0000","last_undegraded":"2023-10-09T06:39:27.412523+0000","last_fullsized":"2023-10-09T06:39:27.412523+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":431,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320746+0000","last_change":"2023-10-09T06:35:45.668825+0000","last_active":"2023-10-09T06:39:27.320746+0000","last_peered":"2023-10-09T06:39:27.320746+0000","last_clean":"2023-10-09T06:39:27.320746+0000","last_became_active":"2023-10-09T06:35:45.668234+0000","last_became_peered":"2023-10-09T06:35:45.668234+0000","last_unstale":"2023-10-09T06:39:27.320746+0000","last_undegraded":"2023-10-09T06:39:27.320746+0000","last_fullsized":"2023-10-09T06:39:27.320746+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":250,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411101+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:39:27.411101+0000","last_peered":"2023-10-09T06:39:27.411101+0000","last_clean":"2023-10-09T06:39:27.411101+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:39:27.411101+0000","last_undegraded":"2023-10-09T06:39:27.411101+0000","last_fullsized":"2023-10-09T06:39:27.411101+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"247'2847","reported_seq":2862,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:39:57.486844+0000","last_change":"2023-10-09T06:39:06.328261+0000","last_active":"2023-10-09T06:39:57.486844+0000","last_peered":"2023-10-09T06:39:57.486844+0000","last_clean":"2023-10-09T06:39:57.486844+0000","last_became_active":"2023-10-09T06:39:06.327897+0000","last_became_peered":"2023-10-09T06:39:06.327897+0000","last_unstale":"2023-10-09T06:39:57.486844+0000","last_undegraded":"2023-10-09T06:39:57.486844+0000","last_fullsized":"2023-10-09T06:39:57.486844+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":2847,"log_dups_size":0,"ondisk_log_size":2847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:24.948782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186580992,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2847,"num_write_kb":182208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":767,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.418025+0000","last_change":"2023-10-09T06:28:34.040634+0000","last_active":"2023-10-09T06:39:27.418025+0000","last_peered":"2023-10-09T06:39:27.418025+0000","last_clean":"2023-10-09T06:39:27.418025+0000","last_became_active":"2023-10-09T06:28:34.039870+0000","last_became_peered":"2023-10-09T06:28:34.039870+0000","last_unstale":"2023-10-09T06:39:27.418025+0000","last_undegraded":"2023-10-09T06:39:27.418025+0000","last_fullsized":"2023-10-09T06:39:27.418025+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":210,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.312101+0000","last_change":"2023-10-09T06:13:43.430584+0000","last_active":"2023-10-09T06:39:27.312101+0000","last_peered":"2023-10-09T06:39:27.312101+0000","last_clean":"2023-10-09T06:39:27.312101+0000","last_became_active":"2023-10-09T06:13:43.430153+0000","last_became_peered":"2023-10-09T06:13:43.430153+0000","last_unstale":"2023-10-09T06:39:27.312101+0000","last_undegraded":"2023-10-09T06:39:27.312101+0000","last_fullsized":"2023-10-09T06:39:27.312101+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:21.697335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.319861+0000","last_change":"2023-10-09T06:38:22.286335+0000","last_active":"2023-10-09T06:39:27.319861+0000","last_peered":"2023-10-09T06:39:27.319861+0000","last_clean":"2023-10-09T06:39:27.319861+0000","last_became_active":"2023-10-09T06:38:22.285968+0000","last_became_peered":"2023-10-09T06:38:22.285968+0000","last_unstale":"2023-10-09T06:39:27.319861+0000","last_undegraded":"2023-10-09T06:39:27.319861+0000","last_fullsized":"2023-10-09T06:39:27.319861+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.311799+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:39:27.311799+0000","last_peered":"2023-10-09T06:39:27.311799+0000","last_clean":"2023-10-09T06:39:27.311799+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:39:27.311799+0000","last_undegraded":"2023-10-09T06:39:27.311799+0000","last_fullsized":"2023-10-09T06:39:27.311799+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412667+0000","last_change":"2023-10-09T06:38:22.272583+0000","last_active":"2023-10-09T06:39:27.412667+0000","last_peered":"2023-10-09T06:39:27.412667+0000","last_clean":"2023-10-09T06:39:27.412667+0000","last_became_active":"2023-10-09T06:38:22.272071+0000","last_became_peered":"2023-10-09T06:38:22.272071+0000","last_unstale":"2023-10-09T06:39:27.412667+0000","last_undegraded":"2023-10-09T06:39:27.412667+0000","last_fullsized":"2023-10-09T06:39:27.412667+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:34.640713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":41,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320586+0000","last_change":"2023-10-09T06:38:22.373881+0000","last_active":"2023-10-09T06:39:27.320586+0000","last_peered":"2023-10-09T06:39:27.320586+0000","last_clean":"2023-10-09T06:39:27.320586+0000","last_became_active":"2023-10-09T06:38:22.276876+0000","last_became_peered":"2023-10-09T06:38:22.276876+0000","last_unstale":"2023-10-09T06:39:27.320586+0000","last_undegraded":"2023-10-09T06:39:27.320586+0000","last_fullsized":"2023-10-09T06:39:27.320586+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320593+0000","last_change":"2023-10-09T06:14:42.688026+0000","last_active":"2023-10-09T06:39:27.320593+0000","last_peered":"2023-10-09T06:39:27.320593+0000","last_clean":"2023-10-09T06:39:27.320593+0000","last_became_active":"2023-10-09T06:14:42.687571+0000","last_became_peered":"2023-10-09T06:14:42.687571+0000","last_unstale":"2023-10-09T06:39:27.320593+0000","last_undegraded":"2023-10-09T06:39:27.320593+0000","last_fullsized":"2023-10-09T06:39:27.320593+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":41,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320606+0000","last_change":"2023-10-09T06:38:22.321430+0000","last_active":"2023-10-09T06:39:27.320606+0000","last_peered":"2023-10-09T06:39:27.320606+0000","last_clean":"2023-10-09T06:39:27.320606+0000","last_became_active":"2023-10-09T06:38:22.273775+0000","last_became_peered":"2023-10-09T06:38:22.273775+0000","last_unstale":"2023-10-09T06:39:27.320606+0000","last_undegraded":"2023-10-09T06:39:27.320606+0000","last_fullsized":"2023-10-09T06:39:27.320606+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":226,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.320173+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:39:27.320173+0000","last_peered":"2023-10-09T06:39:27.320173+0000","last_clean":"2023-10-09T06:39:27.320173+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:39:27.320173+0000","last_undegraded":"2023-10-09T06:39:27.320173+0000","last_fullsized":"2023-10-09T06:39:27.320173+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":251,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.412218+0000","last_change":"2023-10-09T06:28:31.016421+0000","last_active":"2023-10-09T06:39:27.412218+0000","last_peered":"2023-10-09T06:39:27.412218+0000","last_clean":"2023-10-09T06:39:27.412218+0000","last_became_active":"2023-10-09T06:28:31.015955+0000","last_became_peered":"2023-10-09T06:28:31.015955+0000","last_unstale":"2023-10-09T06:39:27.412218+0000","last_undegraded":"2023-10-09T06:39:27.412218+0000","last_fullsized":"2023-10-09T06:39:27.412218+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:00.304906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.411515+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:39:27.411515+0000","last_peered":"2023-10-09T06:39:27.411515+0000","last_clean":"2023-10-09T06:39:27.411515+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:39:27.411515+0000","last_undegraded":"2023-10-09T06:39:27.411515+0000","last_fullsized":"2023-10-09T06:39:27.411515+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":225,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.312916+0000","last_change":"2023-10-09T06:38:22.366125+0000","last_active":"2023-10-09T06:39:27.312916+0000","last_peered":"2023-10-09T06:39:27.312916+0000","last_clean":"2023-10-09T06:39:27.312916+0000","last_became_active":"2023-10-09T06:38:22.273346+0000","last_became_peered":"2023-10-09T06:38:22.273346+0000","last_unstale":"2023-10-09T06:39:27.312916+0000","last_undegraded":"2023-10-09T06:39:27.312916+0000","last_fullsized":"2023-10-09T06:39:27.312916+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:16:17.145372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.319862+0000","last_change":"2023-10-09T06:28:31.029311+0000","last_active":"2023-10-09T06:39:27.319862+0000","last_peered":"2023-10-09T06:39:27.319862+0000","last_clean":"2023-10-09T06:39:27.319862+0000","last_became_active":"2023-10-09T06:28:31.028931+0000","last_became_peered":"2023-10-09T06:28:31.028931+0000","last_unstale":"2023-10-09T06:39:27.319862+0000","last_undegraded":"2023-10-09T06:39:27.319862+0000","last_fullsized":"2023-10-09T06:39:27.319862+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19916,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2976579584,"num_objects":45419,"num_object_clones":0,"num_object_copies":136257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45419,"num_write_kb":2906816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8887795712,"data_stored":8887795712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45419,"ondisk_log_size":45419,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5928,"num_read_kb":12158,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430704,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2458368,"kb_used_data":1868664,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91286784,"statfs":{"total":95995035648,"available":93477666816,"internally_reserved":0,"allocated":1913511936,"data_stored":1912922353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,0,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.252,"15min":2.5369999999999999},"min":{"1min":0.51600000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.3460000000000001,"5min":5.4699999999999998,"15min":178.738},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.0780000000000001,"15min":2.4220000000000002},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.252},"max":{"1min":1.046,"5min":5.8399999999999999,"15min":177.75},"last":0.68000000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.504,"15min":1.857},"min":{"1min":0.54800000000000004,"5min":0.27800000000000002,"15min":0.247},"max":{"1min":1.2609999999999999,"5min":9.3949999999999996,"15min":148.88499999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.5780000000000001,"15min":1.923},"min":{"1min":0.53500000000000003,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.244,"5min":13.922000000000001,"15min":148.059},"last":1.016}]},{"osd":2,"last update":"Mon Oct 9 06:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.7769999999999999,"15min":1.6040000000000001},"min":{"1min":0.54600000000000004,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.3159999999999998,"5min":15.957000000000001,"15min":15.957000000000001},"last":3.9180000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.4179999999999999,"15min":1.405},"min":{"1min":0.501,"5min":0.34100000000000003,"15min":0.307},"max":{"1min":2.4489999999999998,"5min":5.6269999999999998,"15min":7.0629999999999997},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:39:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.335,"15min":3.335},"min":{"1min":0.66000000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5069999999999999,"5min":28.376999999999999,"15min":183.971},"last":6.0750000000000002},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.8879999999999999,"15min":3.073},"min":{"1min":0.67800000000000005,"5min":0.376,"15min":0.33200000000000002},"max":{"1min":1.53,"5min":28.280999999999999,"15min":183.095},"last":0.57499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.302,"15min":1.0740000000000001},"min":{"1min":0.68100000000000005,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":1.2210000000000001,"5min":12.382999999999999,"15min":12.382999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.845,"15min":1.5640000000000001},"min":{"1min":0.85799999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.415,"5min":16.064,"15min":16.064},"last":6.0190000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.3799999999999999,"15min":1.2509999999999999},"min":{"1min":0.70999999999999996,"5min":0.34799999999999998,"15min":0.28199999999999997},"max":{"1min":1.385,"5min":10.680999999999999,"15min":10.680999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.73,"15min":1.387},"min":{"1min":0.63100000000000001,"5min":0.33900000000000002,"15min":0.33100000000000002},"max":{"1min":1.2549999999999999,"5min":11.444000000000001,"15min":11.444000000000001},"last":0.76500000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.3120000000000001,"15min":1.298},"min":{"1min":0.82599999999999996,"5min":0.36799999999999999,"15min":0.314},"max":{"1min":1.353,"5min":8.2439999999999998,"15min":15.521000000000001},"last":1.143},{"interface":"front","average":{"1min":1.129,"5min":1.8520000000000001,"15min":1.5660000000000001},"min":{"1min":0.80200000000000005,"5min":0.378,"15min":0.33400000000000002},"max":{"1min":1.738,"5min":16.024000000000001,"15min":16.024000000000001},"last":3.992}]}]},{"osd":6,"up_from":42,"seq":180388626936,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1034932,"kb_used_data":733612,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92710220,"statfs":{"total":95995035648,"available":94935265280,"internally_reserved":0,"allocated":751218688,"data_stored":750641190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.2949999999999999,"15min":1.244},"min":{"1min":0.53500000000000003,"5min":0.245,"15min":0.245},"max":{"1min":1.29,"5min":8.6280000000000001,"15min":9.4649999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.532,"15min":1.466},"min":{"1min":0.67500000000000004,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":1.2909999999999999,"5min":10.696,"15min":10.696},"last":0.36299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.698,"15min":2.3799999999999999},"min":{"1min":0.51800000000000002,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":1.1499999999999999,"5min":19.25,"15min":316.88099999999997},"last":1.204},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.5600000000000001,"15min":2.2989999999999999},"min":{"1min":0.374,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.169,"5min":7.952,"15min":316.84399999999999},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.452,"15min":1.8799999999999999},"min":{"1min":0.64800000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.6160000000000001,"5min":7.4500000000000002,"15min":53.616},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.026,"5min":1.6659999999999999,"15min":1.9339999999999999},"min":{"1min":0.48799999999999999,"5min":0.30599999999999999,"15min":0.29799999999999999},"max":{"1min":3.7530000000000001,"5min":7.3390000000000004,"15min":53.665999999999997},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:39:11 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.8819999999999999,"15min":2.4350000000000001},"min":{"1min":0.68600000000000005,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":1.492,"5min":8.7210000000000001,"15min":115.09399999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.871,"5min":1.8480000000000001,"15min":2.1840000000000002},"min":{"1min":0.57899999999999996,"5min":0.36799999999999999,"15min":0.32000000000000001},"max":{"1min":1.097,"5min":11.512,"15min":112.886},"last":0.65300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.585,"15min":1.252},"min":{"1min":0.60299999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.3460000000000001,"5min":10.653,"15min":10.653},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.718,"15min":1.429},"min":{"1min":0.44,"5min":0.317,"15min":0.29799999999999999},"max":{"1min":1.339,"5min":16.065000000000001,"15min":16.065000000000001},"last":0.78200000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.587,"15min":1.3480000000000001},"min":{"1min":0.66500000000000004,"5min":0.36099999999999999,"15min":0.34300000000000003},"max":{"1min":1.391,"5min":10.601000000000001,"15min":10.601000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":2.2589999999999999,"15min":1.748},"min":{"1min":0.58799999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.399,"5min":11.526,"15min":12.215999999999999},"last":0.60599999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.4379999999999999,"15min":1.3959999999999999},"min":{"1min":0.80200000000000005,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":1.7649999999999999,"5min":10.673,"15min":11.076000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.147,"5min":1.385,"15min":1.2509999999999999},"min":{"1min":0.78000000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.639,"5min":9.0459999999999994,"15min":10.74},"last":0.59799999999999998}]}]},{"osd":1,"up_from":93,"seq":399431958845,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307136,"kb_used_data":1757816,"kb_used_omap":7,"kb_used_meta":549304,"kb_avail":91438016,"statfs":{"total":95995035648,"available":93632528384,"internally_reserved":0,"allocated":1800003584,"data_stored":1799008375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":562487696},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":1.556,"15min":2.6320000000000001},"min":{"1min":0.29699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":1.042,"5min":9.0449999999999999,"15min":132.886},"last":0.253},{"interface":"front","average":{"1min":0.55000000000000004,"5min":2.2650000000000001,"15min":2.931},"min":{"1min":0.21299999999999999,"5min":0.184,"15min":0.184},"max":{"1min":0.93799999999999994,"5min":14.74,"15min":133.315},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":2.1699999999999999,"15min":2.1840000000000002},"min":{"1min":0.33700000000000002,"5min":0.224,"15min":0.19800000000000001},"max":{"1min":1.181,"5min":16.734000000000002,"15min":67.206000000000003},"last":1.161},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.1230000000000002,"15min":2.23},"min":{"1min":0.42499999999999999,"5min":0.23699999999999999,"15min":0.224},"max":{"1min":1.3149999999999999,"5min":16.654,"15min":67.132000000000005},"last":0.80700000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":4.1390000000000002,"15min":4.71},"min":{"1min":0.41499999999999998,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":1.3759999999999999,"5min":131.99600000000001,"15min":132.792},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":4.1340000000000003,"15min":4.6799999999999997},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":1.073,"5min":134.25299999999999,"15min":134.25299999999999},"last":0.28799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.5149999999999999,"15min":1.52},"min":{"1min":0.64700000000000002,"5min":0.34799999999999998,"15min":0.32600000000000001},"max":{"1min":1.8109999999999999,"5min":16.617999999999999,"15min":26.068999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.3580000000000001,"15min":1.4490000000000001},"min":{"1min":0.63600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.1699999999999999,"5min":8.9169999999999998,"15min":12.907999999999999},"last":0.316}]},{"osd":5,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":2.2570000000000001,"15min":1.7989999999999999},"min":{"1min":0.55600000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.381,"5min":11.436999999999999,"15min":14.170999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.641,"15min":1.6140000000000001},"min":{"1min":0.56699999999999995,"5min":0.40400000000000003,"15min":0.36499999999999999},"max":{"1min":1.9550000000000001,"5min":16.702999999999999,"15min":26.157},"last":0.44600000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.968,"15min":1.74},"min":{"1min":0.51800000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.4410000000000001,"5min":8.1389999999999993,"15min":14.212999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.9450000000000001,"15min":1.823},"min":{"1min":0.67400000000000004,"5min":0.38700000000000001,"15min":0.34999999999999998},"max":{"1min":2.0049999999999999,"5min":16.670000000000002,"15min":26.132000000000001},"last":5.0129999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.7390000000000001,"15min":1.8859999999999999},"min":{"1min":0.59699999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.0590000000000002,"5min":16.687999999999999,"15min":30.774999999999999},"last":5.2530000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.482,"15min":1.599},"min":{"1min":0.60099999999999998,"5min":0.41899999999999998,"15min":0.38600000000000001},"max":{"1min":1.3580000000000001,"5min":10.025,"15min":30.815000000000001},"last":0.59799999999999998}]}]},{"osd":0,"up_from":88,"seq":377957122371,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1329640,"kb_used_data":742112,"kb_used_omap":8,"kb_used_meta":587511,"kb_avail":92415512,"statfs":{"total":95995035648,"available":94633484288,"internally_reserved":0,"allocated":759922688,"data_stored":758948454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":601611520},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":2.1379999999999999,"15min":2.7400000000000002},"min":{"1min":0.34999999999999998,"5min":0.193,"15min":0.193},"max":{"1min":0.95699999999999996,"5min":25.084,"15min":138.31700000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":2.0840000000000001,"15min":2.4870000000000001},"min":{"1min":0.40300000000000002,"5min":0.20899999999999999,"15min":0.20200000000000001},"max":{"1min":0.97699999999999998,"5min":25.036999999999999,"15min":144.37},"last":0.48399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":1.3740000000000001,"15min":1.9470000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.185},"max":{"1min":2.484,"5min":10.076000000000001,"15min":28.745000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.476,"15min":1.8280000000000001},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.18099999999999999},"max":{"1min":5.46,"5min":15.351000000000001,"15min":28.146999999999998},"last":0.66000000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":3.1890000000000001,"15min":2.9100000000000001},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":0.84099999999999997,"5min":86.709999999999994,"15min":86.709999999999994},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.68600000000000005,"5min":3.0190000000000001,"15min":2.7530000000000001},"min":{"1min":0.307,"5min":0.24299999999999999,"15min":0.23699999999999999},"max":{"1min":1.224,"5min":86.641999999999996,"15min":86.641999999999996},"last":1.242}]},{"osd":4,"last update":"Mon Oct 9 06:39:17 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.579,"15min":1.49},"min":{"1min":0.622,"5min":0.309,"15min":0.30299999999999999},"max":{"1min":1.2410000000000001,"5min":9.4209999999999994,"15min":9.4209999999999994},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.873,"5min":1.768,"15min":1.796},"min":{"1min":0.57999999999999996,"5min":0.317,"15min":0.317},"max":{"1min":1.05,"5min":10.238,"15min":28.629000000000001},"last":0.40500000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:39:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":2.0489999999999999,"15min":2.0369999999999999},"min":{"1min":0.75700000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.3340000000000001,"5min":10.151,"15min":28.574000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.7929999999999999,"15min":1.802},"min":{"1min":0.59699999999999998,"5min":0.34100000000000003,"15min":0.32600000000000001},"max":{"1min":1.395,"5min":10.419,"15min":28.061},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:39:17 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.7170000000000001,"15min":1.794},"min":{"1min":0.56599999999999995,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.302,"5min":10.377000000000001,"15min":27.966999999999999},"last":0.373},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.675,"15min":1.5249999999999999},"min":{"1min":0.64800000000000002,"5min":0.38900000000000001,"15min":0.33700000000000002},"max":{"1min":1.325,"5min":9.4510000000000005,"15min":9.4510000000000005},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:39:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":2.1269999999999998,"15min":1.9690000000000001},"min":{"1min":0.52900000000000003,"5min":0.432,"15min":0.40300000000000002},"max":{"1min":1.3200000000000001,"5min":13.259,"15min":28.109999999999999},"last":1.042},{"interface":"front","average":{"1min":1.0629999999999999,"5min":2.02,"15min":1.875},"min":{"1min":0.72199999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.387,"5min":13.428000000000001,"15min":28.074999999999999},"last":1.087}]}]},{"osd":2,"up_from":98,"seq":420906795319,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1587880,"kb_used_data":1016224,"kb_used_omap":11,"kb_used_meta":571636,"kb_avail":92157272,"statfs":{"total":95995035648,"available":94369046528,"internally_reserved":0,"allocated":1040613376,"data_stored":1039636711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":585356222},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":3.4319999999999999,"15min":2.6890000000000001},"min":{"1min":0.372,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.79600000000000004,"5min":64.356999999999999,"15min":117.2},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":3.2469999999999999,"15min":2.6299999999999999},"min":{"1min":0.246,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":0.82699999999999996,"5min":63.968000000000004,"15min":121.521},"last":0.218}]},{"osd":1,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":2.9729999999999999,"15min":2.3370000000000002},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.17999999999999999},"max":{"1min":1.01,"5min":57.802,"15min":60.155000000000001},"last":0.20499999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.5550000000000002,"15min":2.2189999999999999},"min":{"1min":0.27500000000000002,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":0.92700000000000005,"5min":53.771999999999998,"15min":62.582999999999998},"last":0.28199999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.2559999999999998,"15min":3.5819999999999999},"min":{"1min":0.33800000000000002,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":1.671,"5min":27.050999999999998,"15min":201.33600000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.69899999999999995,"5min":2.1709999999999998,"15min":3.512},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.21299999999999999},"max":{"1min":1.5169999999999999,"5min":32.167000000000002,"15min":199.95099999999999},"last":2.3780000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":1.8580000000000001,"15min":1.746},"min":{"1min":0.46899999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.131,"5min":32.433,"15min":32.433},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.643,"15min":1.575},"min":{"1min":0.54000000000000004,"5min":0.35099999999999998,"15min":0.34200000000000003},"max":{"1min":1.083,"5min":32.286999999999999,"15min":32.286999999999999},"last":0.33200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":2.0510000000000002,"15min":2.2759999999999998},"min":{"1min":0.52700000000000002,"5min":0.34399999999999997,"15min":0.33900000000000002},"max":{"1min":1.1859999999999999,"5min":32.475999999999999,"15min":86.423000000000002},"last":0.376},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.9930000000000001,"15min":2.004},"min":{"1min":0.436,"5min":0.38300000000000001,"15min":0.33800000000000002},"max":{"1min":1.004,"5min":32.331000000000003,"15min":86.382999999999996},"last":0.41799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.722,"15min":1.681},"min":{"1min":0.48299999999999998,"5min":0.35199999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":32.357999999999997,"15min":32.357999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.88,"5min":2.21,"15min":1.7989999999999999},"min":{"1min":0.42199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1659999999999999,"5min":32.377000000000002,"15min":32.377000000000002},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.1579999999999999,"15min":1.8620000000000001},"min":{"1min":0.66200000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.238,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":2.0259999999999998,"15min":1.915},"min":{"1min":0.57699999999999996,"5min":0.434,"15min":0.35399999999999998},"max":{"1min":1.286,"5min":32.505000000000003,"15min":32.505000000000003},"last":0.51600000000000001}]}]},{"osd":3,"up_from":103,"seq":442381631793,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2325512,"kb_used_data":1735680,"kb_used_omap":3,"kb_used_meta":589820,"kb_avail":91419640,"statfs":{"total":95995035648,"available":93613711360,"internally_reserved":0,"allocated":1777336320,"data_stored":1776329771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":603976225},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.3570000000000002,"15min":2.734},"min":{"1min":0.28399999999999997,"5min":0.218,"15min":0.18099999999999999},"max":{"1min":0.94299999999999995,"5min":49.280000000000001,"15min":147.988},"last":1.851},{"interface":"front","average":{"1min":0.74399999999999999,"5min":1.9870000000000001,"15min":2.379},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.18099999999999999},"max":{"1min":4.4009999999999998,"5min":26.988,"15min":147.94200000000001},"last":0.314}]},{"osd":1,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.464,"15min":1.8420000000000001},"min":{"1min":0.28199999999999997,"5min":0.21199999999999999,"15min":0.19},"max":{"1min":0.91600000000000004,"5min":9.5609999999999999,"15min":47.057000000000002},"last":1.889},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.466,"15min":2.1880000000000002},"min":{"1min":0.36299999999999999,"5min":0.255,"15min":0.218},"max":{"1min":1.0640000000000001,"5min":7.0860000000000003,"15min":54.752000000000002},"last":1.0109999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.4510000000000001,"15min":2.23},"min":{"1min":0.36799999999999999,"5min":0.28599999999999998,"15min":0.20599999999999999},"max":{"1min":1.1559999999999999,"5min":10.787000000000001,"15min":47.097999999999999},"last":1.823},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.274,"15min":1.796},"min":{"1min":0.34499999999999997,"5min":0.252,"15min":0.23499999999999999},"max":{"1min":1.403,"5min":6.1109999999999998,"15min":46.997},"last":1.913}]},{"osd":4,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.7629999999999999,"15min":1.946},"min":{"1min":0.63500000000000001,"5min":0.34499999999999997,"15min":0.30399999999999999},"max":{"1min":1.123,"5min":13.846,"15min":72.653000000000006},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.4079999999999999,"15min":1.9410000000000001},"min":{"1min":0.60499999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.4930000000000001,"5min":6.8140000000000001,"15min":72.706999999999994},"last":0.52700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.4259999999999999,"15min":1.6839999999999999},"min":{"1min":0.56799999999999995,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.5660000000000001,"5min":8.9120000000000008,"15min":11.781000000000001},"last":1.9339999999999999},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.9410000000000001,"15min":1.861},"min":{"1min":0.58499999999999996,"5min":0.35899999999999999,"15min":0.32300000000000001},"max":{"1min":1.2270000000000001,"5min":10.315,"15min":11.279},"last":0.84999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.409,"15min":1.796},"min":{"1min":0.70199999999999996,"5min":0.40400000000000003,"15min":0.38900000000000001},"max":{"1min":1.6220000000000001,"5min":7.984,"15min":15.122999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.474,"15min":1.671},"min":{"1min":0.58199999999999996,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.6519999999999999,"5min":8.8529999999999998,"15min":11.728999999999999},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:31 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.964,"15min":1.9710000000000001},"min":{"1min":0.71499999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1919999999999999,"5min":9.3469999999999995,"15min":18.695},"last":0.87},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.5960000000000001,"15min":1.952},"min":{"1min":0.67900000000000005,"5min":0.41199999999999998,"15min":0.40200000000000002},"max":{"1min":1.7490000000000001,"5min":9.9550000000000001,"15min":13.211},"last":2.2869999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019402,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333496,"kb_used_data":743792,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92411656,"statfs":{"total":95995035648,"available":94629535744,"internally_reserved":0,"allocated":761643008,"data_stored":761061003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":2.7370000000000001,"15min":2.214},"min":{"1min":0.67300000000000004,"5min":0.249,"15min":0.249},"max":{"1min":1.2649999999999999,"5min":61.069000000000003,"15min":115.59699999999999},"last":0.316},{"interface":"front","average":{"1min":0.94099999999999995,"5min":2.8290000000000002,"15min":2.4649999999999999},"min":{"1min":0.66500000000000004,"5min":0.32600000000000001,"15min":0.27700000000000002},"max":{"1min":1.206,"5min":61.106999999999999,"15min":115.65000000000001},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.4370000000000001,"15min":1.966},"min":{"1min":0.51600000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":1.669,"5min":10.553000000000001,"15min":179.47300000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.554,"15min":2.198},"min":{"1min":0.54900000000000004,"5min":0.32500000000000001,"15min":0.28499999999999998},"max":{"1min":1.4359999999999999,"5min":9.1129999999999995,"15min":179.447},"last":0.72899999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":1.9279999999999999,"15min":1.6830000000000001},"min":{"1min":0.56200000000000006,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":1.173,"5min":24.068000000000001,"15min":24.068000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.952,"15min":1.7490000000000001},"min":{"1min":0.46300000000000002,"5min":0.35899999999999999,"15min":0.29899999999999999},"max":{"1min":1.2190000000000001,"5min":23.434000000000001,"15min":23.434000000000001},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":3.75,"15min":2.6760000000000002},"min":{"1min":0.51200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.3480000000000001,"5min":98.364000000000004,"15min":98.364000000000004},"last":2.7170000000000001},{"interface":"front","average":{"1min":1.036,"5min":3.8180000000000001,"15min":2.4910000000000001},"min":{"1min":0.499,"5min":0.33900000000000002,"15min":0.314},"max":{"1min":1.3959999999999999,"5min":98.296999999999997,"15min":98.296999999999997},"last":2.254}]},{"osd":5,"last update":"Mon Oct 9 06:39:41 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.7050000000000001,"15min":1.244},"min":{"1min":0.42099999999999999,"5min":0.33500000000000002,"15min":0.28499999999999998},"max":{"1min":1.601,"5min":22.068000000000001,"15min":22.068000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.05,"5min":1.6739999999999999,"15min":1.46},"min":{"1min":0.55400000000000005,"5min":0.34699999999999998,"15min":0.26500000000000001},"max":{"1min":1.758,"5min":13.095000000000001,"15min":13.095000000000001},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.595,"15min":1.391},"min":{"1min":0.79200000000000004,"5min":0.39100000000000001,"15min":0.317},"max":{"1min":1.3759999999999999,"5min":7.9020000000000001,"15min":8.1110000000000007},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.149,"5min":1.3260000000000001,"15min":1.3260000000000001},"min":{"1min":0.79700000000000004,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.542,"5min":6.2320000000000002,"15min":11.782},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.49,"15min":1.504},"min":{"1min":0.90200000000000002,"5min":0.372,"15min":0.371},"max":{"1min":1.6339999999999999,"5min":7.3550000000000004,"15min":12.779},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.5209999999999999,"15min":1.4750000000000001},"min":{"1min":0.56999999999999995,"5min":0.377,"15min":0.317},"max":{"1min":2.3999999999999999,"5min":7.835,"15min":14.896000000000001},"last":0.88900000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823169,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":760217600,"data_stored":760217600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1819475968,"data_stored":1819475968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1036124160,"data_stored":1036124160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1821376512,"data_stored":1821376512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":774045696,"data_stored":774045696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":749010944,"data_stored":749010944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1927544832,"data_stored":1927544832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:40:07.036 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T06:40:07.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:07.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:07.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:07.240 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:07.283 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:07.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:07.416 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:07.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:07.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.321658+0000 mon.a (mon.0) 1310 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:07.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.321690+0000 mon.a (mon.0) 1311 : cluster 1 Cluster is now healthy 2023-10-09T06:40:07.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: audit 2023-10-09T06:40:06.331337+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:07.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.331398+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:40:07.532 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.334165+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.388810+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.391141+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.394145+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.396520+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[80002]: cluster 2023-10-09T06:40:06.457500+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.533 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.321658+0000 mon.a (mon.0) 1310 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.321690+0000 mon.a (mon.0) 1311 : cluster 1 Cluster is now healthy 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: audit 2023-10-09T06:40:06.331337+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.331398+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.334165+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.388810+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.391141+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.394145+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.396520+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.534 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:07 smithi125 bash[77299]: cluster 2023-10-09T06:40:06.457500+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.321658+0000 mon.a (mon.0) 1310 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.321690+0000 mon.a (mon.0) 1311 : cluster 1 Cluster is now healthy 2023-10-09T06:40:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: audit 2023-10-09T06:40:06.331337+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.125:0/1010982811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.331398+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.334165+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.388810+0000 osd.4 (osd.4) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.391141+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.394145+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.396520+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:07 smithi155 bash[59495]: cluster 2023-10-09T06:40:06.457500+0000 osd.5 (osd.5) 191 : cluster 3 failed to encode map e248 with expected crc 2023-10-09T06:40:07.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:07.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:07.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:07.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:07.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:07.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 56018 56002 97.2149 44 0.0118444 0.0102806 2023-10-09T06:40:07.950 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:07.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:08.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.221 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:08.349 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:08 smithi125 bash[77299]: cluster 2023-10-09T06:40:07.258064+0000 mgr.y (mgr.25723) 1065 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:40:08.349 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:08 smithi125 bash[80002]: cluster 2023-10-09T06:40:07.258064+0000 mgr.y (mgr.25723) 1065 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:40:08.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:08.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:08.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:08.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:08 smithi155 bash[59495]: cluster 2023-10-09T06:40:07.258064+0000 mgr.y (mgr.25723) 1065 : cluster 0 pgmap v1121: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T06:40:08.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.904 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:08.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 57213 57197 96.6059 74.6875 0.0146205 0.0103448 2023-10-09T06:40:09.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:09.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:09.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:09.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:09.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:09.303 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:09.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:09.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:09.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:09.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:09.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.870 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:09.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:09.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 58352 58336 95.9368 71.1875 0.00395703 0.0104162 2023-10-09T06:40:09.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:10.028 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:10 smithi125 bash[77299]: cluster 2023-10-09T06:40:09.258933+0000 mgr.y (mgr.25723) 1066 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:40:10.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:10.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:10 smithi155 bash[59495]: cluster 2023-10-09T06:40:09.258933+0000 mgr.y (mgr.25723) 1066 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:40:10.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:10.367 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:10 smithi125 bash[80002]: cluster 2023-10-09T06:40:09.258933+0000 mgr.y (mgr.25723) 1066 : cluster 0 pgmap v1122: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:40:10.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:10.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:10.535 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:10.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:10.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:10.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:10.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:10.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:10.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:10.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 59511 59495 95.3341 72.4375 0.0303688 0.0104824 2023-10-09T06:40:11.037 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:11.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:40:11.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:11.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:40:11.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:11.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:11.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:11.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:11.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:11.390 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:11] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-10-09T06:40:11.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:11.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:11.703 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:11.704 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:11.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:11.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:11.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:11.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:11.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:40:11.937799+0000 min lat: 0.00126636 max lat: 0.383876 avg lat: 0.0105718 2023-10-09T06:40:11.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:11.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 60518 60502 94.524 62.9375 0.00796098 0.0105718 2023-10-09T06:40:12.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:12.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:12.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:12.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:12.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:12.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:12.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:12.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:12.439 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:12 smithi125 bash[77299]: cluster 2023-10-09T06:40:11.260610+0000 mgr.y (mgr.25723) 1067 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:40:12.440 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:12 smithi125 bash[80002]: cluster 2023-10-09T06:40:11.260610+0000 mgr.y (mgr.25723) 1067 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:40:12.461 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:12.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:12.590 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:12 smithi155 bash[59495]: cluster 2023-10-09T06:40:11.260610+0000 mgr.y (mgr.25723) 1067 : cluster 0 pgmap v1123: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:40:12.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:12.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:12.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:12.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:12.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 61573 61557 93.8266 65.9375 0.00589221 0.0106507 2023-10-09T06:40:12.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:12.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:12.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:13.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:13.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:13.224 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:13.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:13.362 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:13.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:13.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:13.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:13.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:13.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:13.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:13.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:13.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:13.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:13.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:13.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 62660 62644 93.21 67.9375 0.00628178 0.0107224 2023-10-09T06:40:14.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:14.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:14.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:14.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:14.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:14.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:14.326 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:14 smithi125 bash[77299]: cluster 2023-10-09T06:40:13.261344+0000 mgr.y (mgr.25723) 1068 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:14.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:14.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:14.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:14.547 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:14 smithi155 bash[59495]: cluster 2023-10-09T06:40:13.261344+0000 mgr.y (mgr.25723) 1068 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:14.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:14.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:14.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:14 smithi125 bash[80002]: cluster 2023-10-09T06:40:13.261344+0000 mgr.y (mgr.25723) 1068 : cluster 0 pgmap v1124: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:14.671 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:14.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:14.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 63722 63706 92.5858 66.375 0.0162946 0.010795 2023-10-09T06:40:14.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:14.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:15.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.108 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:15.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:15.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:15.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:15.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:15.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:15.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:15.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:15.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:15.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:15.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.773 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:15.876 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:15.902 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:15.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:15.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 16 64788 64772 91.9957 66.625 0.0167951 0.0108647 2023-10-09T06:40:16.005 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:16.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:16.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:16.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:16.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:16.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:16.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:16 smithi155 bash[59495]: cluster 2023-10-09T06:40:15.262064+0000 mgr.y (mgr.25723) 1069 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:16.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:16.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.550 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:16 smithi125 bash[77299]: cluster 2023-10-09T06:40:15.262064+0000 mgr.y (mgr.25723) 1069 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:16.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:16 smithi125 bash[80002]: cluster 2023-10-09T06:40:15.262064+0000 mgr.y (mgr.25723) 1069 : cluster 0 pgmap v1125: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:16.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:16.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:16.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:16.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:16.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:16.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.943 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 65892 65876 91.4845 69 0.00464196 0.0109249 2023-10-09T06:40:16.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:16.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:17.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:17.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:17.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:17.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:17.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:17.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:17.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:17.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:17.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.611 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:17.611 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:17.621 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:17.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:17.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:17.859 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:17.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:17.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 66901 66885 90.8665 63.0625 0.00952015 0.0109992 2023-10-09T06:40:17.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:18.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:18.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:18.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:18.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:18.263 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:18.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:18.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:18.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:18.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:18.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:18.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:18 smithi125 bash[80002]: cluster 2023-10-09T06:40:17.264223+0000 mgr.y (mgr.25723) 1070 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:40:18.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:18 smithi125 bash[77299]: cluster 2023-10-09T06:40:17.264223+0000 mgr.y (mgr.25723) 1070 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:40:18.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:18.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:18 smithi155 bash[59495]: cluster 2023-10-09T06:40:17.264223+0000 mgr.y (mgr.25723) 1070 : cluster 0 pgmap v1126: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:40:18.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:18.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:18.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:18.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:18.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:18.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 67969 67953 90.3533 66.75 0.0178714 0.0110621 2023-10-09T06:40:18.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:18.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:19.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:19.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:19.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:19.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:19.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:19.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:19.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:19.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:19.491 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:19.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:19.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[77299]: audit 2023-10-09T06:40:18.604027+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:19.628 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:19.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:19.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:19.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:19.876 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:19 smithi155 bash[59495]: audit 2023-10-09T06:40:18.604027+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:19.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:19 smithi155 bash[59495]: audit 2023-10-09T06:40:18.606650+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:19.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:19 smithi155 bash[59495]: cluster 2023-10-09T06:40:19.264700+0000 mgr.y (mgr.25723) 1071 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:40:19.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:19.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:19.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[80002]: audit 2023-10-09T06:40:18.604027+0000 mon.a (mon.0) 1314 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:19.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[80002]: audit 2023-10-09T06:40:18.606650+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:19.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[80002]: cluster 2023-10-09T06:40:19.264700+0000 mgr.y (mgr.25723) 1071 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:40:19.895 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[77299]: audit 2023-10-09T06:40:18.606650+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:19.895 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:19 smithi125 bash[77299]: cluster 2023-10-09T06:40:19.264700+0000 mgr.y (mgr.25723) 1071 : cluster 0 pgmap v1127: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:40:19.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 69056 69040 89.8862 67.9375 0.00959207 0.0111185 2023-10-09T06:40:19.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:20.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:20.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:20.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:20.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:20.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:20.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:20.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:20.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.817 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:20.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 70103 70087 89.387 65.4375 0.00574931 0.0111806 2023-10-09T06:40:20.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:20.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:21.055 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:21.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:21.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:21.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:21.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:21.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:21.347 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:21.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:21.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:21 smithi155 bash[59495]: audit 2023-10-09T06:40:21.165293+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:40:21.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:21.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:21.483 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:21] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-10-09T06:40:21.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:21 smithi125 bash[80002]: audit 2023-10-09T06:40:21.165293+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:40:21.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:21 smithi125 bash[77299]: audit 2023-10-09T06:40:21.165293+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:40:21.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:21.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:21.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:21.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:21.749 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:21.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:21.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:21.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:21.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:21.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 71105 71089 88.8516 62.625 0.0248872 0.0112487 2023-10-09T06:40:21.989 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:22.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:22.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:22.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:22.251 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:22 smithi125 bash[77299]: cluster 2023-10-09T06:40:21.265453+0000 mgr.y (mgr.25723) 1072 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:40:22.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:22.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:22.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:22 smithi155 bash[59495]: cluster 2023-10-09T06:40:21.265453+0000 mgr.y (mgr.25723) 1072 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:40:22.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:22.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:22.523 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:22 smithi125 bash[80002]: cluster 2023-10-09T06:40:21.265453+0000 mgr.y (mgr.25723) 1072 : cluster 0 pgmap v1128: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:40:22.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:22.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:22.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:22.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:22.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.934 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:22.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:22.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 72083 72067 88.3079 61.125 0.0148759 0.0113179 2023-10-09T06:40:23.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:23.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:23.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:23.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:23.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:23.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:23.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:23.501 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:40:23.501 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:40:23.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:23.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:23.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:23.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:40:23.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:23.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:23.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:23.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 72777 72761 87.4437 43.375 0.00532977 0.011429 2023-10-09T06:40:23.990 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:24.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:24.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:24.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:24.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:24.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:24.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:24.329 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[80002]: cluster 2023-10-09T06:40:23.265991+0000 mgr.y (mgr.25723) 1073 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:24.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:24.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[77299]: cluster 2023-10-09T06:40:23.265991+0000 mgr.y (mgr.25723) 1073 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:24.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[77299]: audit 2023-10-09T06:40:23.755008+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.125:0/2584610608' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.427 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[77299]: audit 2023-10-09T06:40:23.755964+0000 mon.a (mon.0) 1317 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.427 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[80002]: audit 2023-10-09T06:40:23.755008+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.125:0/2584610608' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.427 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:24 smithi125 bash[80002]: audit 2023-10-09T06:40:23.755964+0000 mon.a (mon.0) 1317 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:24.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:24.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:24 smithi155 bash[59495]: cluster 2023-10-09T06:40:23.265991+0000 mgr.y (mgr.25723) 1073 : cluster 0 pgmap v1129: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:24 smithi155 bash[59495]: audit 2023-10-09T06:40:23.755008+0000 mon.c (mon.2) 100 : audit 1 from='client.? 172.21.15.125:0/2584610608' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:24 smithi155 bash[59495]: audit 2023-10-09T06:40:23.755964+0000 mon.a (mon.0) 1317 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:40:24.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:24.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:24.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:24.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:24.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:24.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:24.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 73805 73789 87.006 64.25 0.0250344 0.0114875 2023-10-09T06:40:24.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:24.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:24.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:25.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:25.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:25.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:25.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:25.681 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:25.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:25.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:25.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:25.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:25.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:25.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 74746 74730 86.4838 58.8125 0.00828838 0.0115557 2023-10-09T06:40:25.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:25.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:26.293 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:26.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:26.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:26.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:26.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:26.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:26.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:26.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:26 smithi155 bash[59495]: cluster 2023-10-09T06:40:25.266573+0000 mgr.y (mgr.25723) 1074 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:26.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:26 smithi125 bash[77299]: cluster 2023-10-09T06:40:25.266573+0000 mgr.y (mgr.25723) 1074 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:26.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:26 smithi125 bash[80002]: cluster 2023-10-09T06:40:25.266573+0000 mgr.y (mgr.25723) 1074 : cluster 0 pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:26.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:26.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:26.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:26.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:26.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:26.944 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 75655 75639 85.9443 56.8125 0.0160496 0.0116297 2023-10-09T06:40:26.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:27.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:27.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:27.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:27.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:27.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:27.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:27.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:27.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:27.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:27.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 76521 76505 85.376 54.125 0.0109397 0.0117076 2023-10-09T06:40:27.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:27.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:28.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:28.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:28.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:28.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:28.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:28.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:28.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:28.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:28.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:28.606 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[77299]: cluster 2023-10-09T06:40:27.267028+0000 mgr.y (mgr.25723) 1075 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:40:28.607 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[77299]: audit 2023-10-09T06:40:27.519828+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.607 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[77299]: audit 2023-10-09T06:40:27.530721+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[80002]: cluster 2023-10-09T06:40:27.267028+0000 mgr.y (mgr.25723) 1075 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:40:28.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[80002]: audit 2023-10-09T06:40:27.519828+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.607 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:28 smithi125 bash[80002]: audit 2023-10-09T06:40:27.530721+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:28 smithi155 bash[59495]: cluster 2023-10-09T06:40:27.267028+0000 mgr.y (mgr.25723) 1075 : cluster 0 pgmap v1131: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-10-09T06:40:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:28 smithi155 bash[59495]: audit 2023-10-09T06:40:27.519828+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:28 smithi155 bash[59495]: audit 2023-10-09T06:40:27.530721+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:28.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:28.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:28.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:28.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:28.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:28.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:28.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 77495 77479 84.9461 60.875 0.0165906 0.0117661 2023-10-09T06:40:29.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:29.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:29.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:29.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:29.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:29.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:29.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:40:29.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:29.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:29.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:29.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:29.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:29.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 78475 78459 84.5374 61.25 0.0519635 0.0118166 2023-10-09T06:40:29.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:29.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:29.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:30.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:40:30.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:30.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:30.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:30.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:30.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:30 smithi125 bash[77299]: cluster 2023-10-09T06:40:29.267573+0000 mgr.y (mgr.25723) 1076 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T06:40:30.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:30 smithi125 bash[80002]: cluster 2023-10-09T06:40:29.267573+0000 mgr.y (mgr.25723) 1076 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T06:40:30.438 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:30 smithi155 bash[59495]: cluster 2023-10-09T06:40:29.267573+0000 mgr.y (mgr.25723) 1076 : cluster 0 pgmap v1132: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s 2023-10-09T06:40:30.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:30.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:30.945 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 79348 79332 84.0293 54.5625 0.0153098 0.0118949 2023-10-09T06:40:31.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:31.067 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:31.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:31.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:31.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:31.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:31.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:31.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:31.447 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:31] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.33.4" 2023-10-09T06:40:31.463 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:31.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:31.585 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:31.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:31.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:31.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:31.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:31.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:31.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:31.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:40:31.940964+0000 min lat: 0.00126636 max lat: 0.383876 avg lat: 0.0119767 2023-10-09T06:40:31.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:31.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 80142 80126 83.4541 49.625 0.00412605 0.0119767 2023-10-09T06:40:31.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:32.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:32.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:32.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:32.290 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:32.383 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:40:32.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.425 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:40:32.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:32.458 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:40:32.499 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:40:32.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:32.538 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:32 smithi155 bash[59495]: cluster 2023-10-09T06:40:31.268148+0000 mgr.y (mgr.25723) 1077 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:40:32.544 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:40:32.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:32.592 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:40:32.635 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:40:32.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:32 smithi125 bash[77299]: cluster 2023-10-09T06:40:31.268148+0000 mgr.y (mgr.25723) 1077 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:40:32.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:32 smithi125 bash[80002]: cluster 2023-10-09T06:40:31.268148+0000 mgr.y (mgr.25723) 1077 : cluster 0 pgmap v1133: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:40:32.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.671 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.675 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.676 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.676 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:40:32.688 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:32.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:32.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:32.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:32.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 80950 80934 82.9138 50.5 0.00726388 0.0120538 2023-10-09T06:40:32.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:32.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:33.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:33.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:33.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:33.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:33.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:33.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:33.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[77299]: audit 2023-10-09T06:40:32.350221+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.125:0/4199140982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:33.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[77299]: audit 2023-10-09T06:40:33.026715+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[77299]: audit 2023-10-09T06:40:33.029457+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.125:0/1977783791' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.351 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:40:33.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:33.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:33.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:33.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:33.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[80002]: audit 2023-10-09T06:40:32.350221+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.125:0/4199140982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:33.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[80002]: audit 2023-10-09T06:40:33.026715+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:33 smithi125 bash[80002]: audit 2023-10-09T06:40:33.029457+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.125:0/1977783791' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.626 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:33 smithi155 bash[59495]: audit 2023-10-09T06:40:32.350221+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.125:0/4199140982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:33 smithi155 bash[59495]: audit 2023-10-09T06:40:33.026715+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:33 smithi155 bash[59495]: audit 2023-10-09T06:40:33.029457+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.125:0/1977783791' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:33.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:33.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:33.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:33.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:33.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:33.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 81851 81835 82.4847 56.3125 0.0139797 0.0121174 2023-10-09T06:40:34.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:34.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:34.134 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:34.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:34.269 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:34.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:34.343 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:40:34.344 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:40:34.344 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:40:34.344 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-10-09T06:40:34.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T06:40:34.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.268714+0000 mgr.y (mgr.25723) 1078 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.322112+0000 mon.a (mon.0) 1321 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: audit 2023-10-09T06:40:33.337257+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.337420+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.346105+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.348890+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: cluster 2023-10-09T06:40:33.532516+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: audit 2023-10-09T06:40:33.604184+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[77299]: audit 2023-10-09T06:40:33.606779+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.268714+0000 mgr.y (mgr.25723) 1078 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.322112+0000 mon.a (mon.0) 1321 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: audit 2023-10-09T06:40:33.337257+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.337420+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:40:34.417 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.346105+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.418 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.348890+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.418 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: cluster 2023-10-09T06:40:33.532516+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.418 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: audit 2023-10-09T06:40:33.604184+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:34.418 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:34 smithi125 bash[80002]: audit 2023-10-09T06:40:33.606779+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:34.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:34.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:34.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:34.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.268714+0000 mgr.y (mgr.25723) 1078 : cluster 0 pgmap v1134: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T06:40:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.322112+0000 mon.a (mon.0) 1321 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: audit 2023-10-09T06:40:33.337257+0000 mon.a (mon.0) 1322 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.337420+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:40:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.346105+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.348890+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: cluster 2023-10-09T06:40:33.532516+0000 osd.5 (osd.5) 192 : cluster 3 failed to encode map e249 with expected crc 2023-10-09T06:40:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: audit 2023-10-09T06:40:33.604184+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:34 smithi155 bash[59495]: audit 2023-10-09T06:40:33.606779+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:34.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:34.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:34.829 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:34.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:34.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 82785 82769 82.1019 58.375 0.0177992 0.0121742 2023-10-09T06:40:34.956 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:34.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:34.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:35.096 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:35.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:35.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:35.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:35.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:35.394 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:35.498 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:35.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:35.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:35.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:35.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:35.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:35.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:35.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:35.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:35.946 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 15 83743 83728 81.7555 59.9375 0.011185 0.0122262 2023-10-09T06:40:36.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:36.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:36.154 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:36.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:36.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:36.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:36.313 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:36.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:36.346 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:36 smithi155 bash[59495]: cluster 2023-10-09T06:40:35.269308+0000 mgr.y (mgr.25723) 1079 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:40:36.424 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:36.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:36.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:36.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:36.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:36 smithi125 bash[77299]: cluster 2023-10-09T06:40:35.269308+0000 mgr.y (mgr.25723) 1079 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:40:36.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:36 smithi125 bash[80002]: cluster 2023-10-09T06:40:35.269308+0000 mgr.y (mgr.25723) 1079 : cluster 0 pgmap v1136: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:40:36.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:36.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:36.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:36.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:36.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:40:36.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:36.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 84658 84642 81.3765 57.125 0.0204954 0.0122826 2023-10-09T06:40:36.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:36.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:37.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:37.124 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:37.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:37.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:37.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:37.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:37.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:37.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:37.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:37.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:37.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:37.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:40:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:37.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:37.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:37.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 85519 85503 80.9588 53.8125 0.0437476 0.0123449 2023-10-09T06:40:37.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:37.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:38.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:38.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:38.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:38.245 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:38.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:38.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:38 smithi125 bash[77299]: cluster 2023-10-09T06:40:37.269782+0000 mgr.y (mgr.25723) 1080 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:40:38.352 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:38 smithi125 bash[80002]: cluster 2023-10-09T06:40:37.269782+0000 mgr.y (mgr.25723) 1080 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:40:38.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:38.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:38.484 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:38 smithi155 bash[59495]: cluster 2023-10-09T06:40:37.269782+0000 mgr.y (mgr.25723) 1080 : cluster 0 pgmap v1137: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:40:38.509 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:38.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:38.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:38.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:38.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:38.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:38.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:38.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:38.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:38.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:38.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 86286 86270 80.4657 47.9375 0.015195 0.012421 2023-10-09T06:40:39.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:39.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:39.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:39.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:39.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:39.333 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:39.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:39.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:39.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:39.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:39.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:39.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:39.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:39.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:39.946 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:39.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 87113 87097 80.0424 51.6875 0.00684516 0.0124856 2023-10-09T06:40:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:39.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:40.036 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:40 smithi125 bash[77299]: cluster 2023-10-09T06:40:39.271578+0000 mgr.y (mgr.25723) 1081 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:40:40.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.083 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:40.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:40.219 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:40.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:40.324 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.335 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:40 smithi155 bash[59495]: cluster 2023-10-09T06:40:39.271578+0000 mgr.y (mgr.25723) 1081 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:40:40.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:40.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:40 smithi125 bash[80002]: cluster 2023-10-09T06:40:39.271578+0000 mgr.y (mgr.25723) 1081 : cluster 0 pgmap v1138: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:40:40.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:40.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:40.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:40.607 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:40.652 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:40.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:40.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:40.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:40.910 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:40.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:40.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 87900 87884 79.5952 49.1875 0.0262599 0.0125586 2023-10-09T06:40:41.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:41.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[77299]: audit 2023-10-09T06:40:40.051282+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[77299]: audit 2023-10-09T06:40:40.060659+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[77299]: audit 2023-10-09T06:40:40.704103+0000 mon.a (mon.0) 1328 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:41.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[77299]: audit 2023-10-09T06:40:40.705356+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:41.058 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[77299]: audit 2023-10-09T06:40:40.711451+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:41.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:41.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.312 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.313 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:41] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.33.4" 2023-10-09T06:40:41.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:41.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[80002]: audit 2023-10-09T06:40:40.051282+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[80002]: audit 2023-10-09T06:40:40.060659+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[80002]: audit 2023-10-09T06:40:40.704103+0000 mon.a (mon.0) 1328 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:41.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[80002]: audit 2023-10-09T06:40:40.705356+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:41.314 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:41 smithi125 bash[80002]: audit 2023-10-09T06:40:40.711451+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.392 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:41.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:41 smithi155 bash[59495]: audit 2023-10-09T06:40:40.051282+0000 mon.a (mon.0) 1326 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:41 smithi155 bash[59495]: audit 2023-10-09T06:40:40.060659+0000 mon.a (mon.0) 1327 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:41 smithi155 bash[59495]: audit 2023-10-09T06:40:40.704103+0000 mon.a (mon.0) 1328 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:41.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:41 smithi155 bash[59495]: audit 2023-10-09T06:40:40.705356+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:41.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:41 smithi155 bash[59495]: audit 2023-10-09T06:40:40.711451+0000 mon.a (mon.0) 1330 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:40:41.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:41.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:41.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:41.720 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:41.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:41.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 88793 88777 79.2554 55.8125 0.0100471 0.0126118 2023-10-09T06:40:41.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:41.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:42.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:42.102 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:42 smithi155 bash[59495]: cluster 2023-10-09T06:40:41.272187+0000 mgr.y (mgr.25723) 1082 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:42.119 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:42.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:42.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:42.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:42.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:42 smithi125 bash[77299]: cluster 2023-10-09T06:40:41.272187+0000 mgr.y (mgr.25723) 1082 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:42.391 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:42 smithi125 bash[80002]: cluster 2023-10-09T06:40:41.272187+0000 mgr.y (mgr.25723) 1082 : cluster 0 pgmap v1139: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:42.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:42.415 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:42.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:42.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:42.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:42.655 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:42.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:42.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:42.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:42.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:42.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 89630 89614 78.8758 52.3125 0.0146709 0.0126724 2023-10-09T06:40:42.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:42.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:42.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:43.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:43.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:43.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:43.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:43.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:43.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:43.374 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:43.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:43.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:43.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:43.643 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:43.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:43.749 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:43.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:43.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:43.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:43.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:43.947 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 90423 90407 78.4686 49.5625 0.00476456 0.0127354 2023-10-09T06:40:44.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:44.064 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:44.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:44.198 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:44.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:44.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:44.335 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:44 smithi125 bash[80002]: cluster 2023-10-09T06:40:43.272804+0000 mgr.y (mgr.25723) 1083 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:40:44.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:44.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:44.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:44.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.589 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:44 smithi125 bash[77299]: cluster 2023-10-09T06:40:43.272804+0000 mgr.y (mgr.25723) 1083 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:40:44.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:44 smithi155 bash[59495]: cluster 2023-10-09T06:40:43.272804+0000 mgr.y (mgr.25723) 1083 : cluster 0 pgmap v1140: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-09T06:40:44.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:44.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:44.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:44.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:44.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 91208 91192 78.0658 49.0625 0.0170926 0.0128045 2023-10-09T06:40:44.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:44.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:44.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:44.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:45.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:45.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:45.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:45.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:45.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:45.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:45.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:45.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:45.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:45.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:91710 9.2 9:4c0be8a0:::benchmark_data_smithi125_199533_object91709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.684309+0000", 2023-10-09T06:40:45.713 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.022270484, 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.022304696999999998, 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91710 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684309+0000", 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684309+0000", 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684308+0000", 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:40:45.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684334+0000", 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6381e-05 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684347+0000", 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2539e-05 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684355+0000", 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8089999999999993e-06 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684530+0000", 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017576000000000001 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684702+0000", 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017117 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.684848+0000", 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014645399999999999 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.701217+0000", 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.016368436 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.701376+0000", 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015916699999999999 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:91715 9.2 9:412475b0:::benchmark_data_smithi125_199533_object91714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.688911+0000", 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.017668728000000002, 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.017816401999999999, 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91715 2023-10-09T06:40:45.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688911+0000", 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688911+0000", 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688910+0000", 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688935+0000", 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5338999999999999e-05 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688950+0000", 2023-10-09T06:40:45.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.499e-05 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.688958+0000", 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4240000000000004e-06 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.689069+0000", 2023-10-09T06:40:45.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011151599999999999 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.689352+0000", 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000282701 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.689702+0000", 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034986899999999999 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.701362+0000", 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011660169 2023-10-09T06:40:45.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.701413+0000", 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1261000000000003e-05 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.727 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45605.0:91720 9.9 e249/241 9:98db1d2b:::benchmark_data_smithi125_199533_object91719:head v 249'5752, mlcod=249'5750)", 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.694691+0000", 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.011889319000000001, 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01213502, 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.694691+0000", 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.694691+0000", 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.694690+0000", 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698022+0000", 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0033324180000000002 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698038+0000", 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5383000000000002e-05 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698045+0000", 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0419999999999997e-06 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698089+0000", 2023-10-09T06:40:45.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4421e-05 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698109+0000", 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0364e-05 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.732 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:91726 9.1 9:88e26136:::benchmark_data_smithi125_199533_object91725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.698620+0000", 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0079603320000000005, 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.008278512, 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91726 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698620+0000", 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698620+0000", 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698618+0000", 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698640+0000", 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2254000000000001e-05 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698658+0000", 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7883000000000001e-05 2023-10-09T06:40:45.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698666+0000", 2023-10-09T06:40:45.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9039999999999992e-06 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.699112+0000", 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000446317 2023-10-09T06:40:45.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.699298+0000", 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185766 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.700228+0000", 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00093008100000000003 2023-10-09T06:40:45.739 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:91727 9.1 9:8425f193:::benchmark_data_smithi125_199533_object91726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.698889+0000", 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0076904620000000003, 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080655699999999993, 2023-10-09T06:40:45.740 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 91727 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698889+0000", 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698889+0000", 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698889+0000", 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698910+0000", 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1472e-05 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698921+0000", 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.135e-05 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.698929+0000", 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9670000000000001e-06 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.703968+0000", 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0050390319999999997 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704104+0000", 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000135549 2023-10-09T06:40:45.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704499+0000", 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00039479000000000001 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45605.0:91724 9.9 e249/241 9:9ff46627:::benchmark_data_smithi125_199533_object91723:head v 249'5753, mlcod=249'5750)", 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.702273+0000", 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0043073800000000004, 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0047256779999999996, 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702273+0000", 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702273+0000", 2023-10-09T06:40:45.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702272+0000", 2023-10-09T06:40:45.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702359+0000", 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7095000000000006e-05 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702424+0000", 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.525e-05 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.702442+0000", 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8097999999999999e-05 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.703955+0000", 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001512273 2023-10-09T06:40:45.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704015+0000", 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.0214000000000003e-05 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop_reply(client.45605.0:91726 9.1 e249/241)", 2023-10-09T06:40:45.753 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.704325+0000", 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0022548080000000001, 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027066740000000001, 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "queued for pg", 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704325+0000", 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704325+0000", 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704324+0000", 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704330+0000", 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4410000000000001e-06 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704354+0000", 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.4334000000000001e-05 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704362+0000", 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6750000000000002e-06 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45605.0:91729 9.3 e249/241 9:c916d354:::benchmark_data_smithi125_199533_object91728:head v 249'5761, mlcod=249'5758)", 2023-10-09T06:40:45.757 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.704442+0000", 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.0021383610000000001, 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026217480000000001, 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704442+0000", 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704442+0000", 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704441+0000", 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704477+0000", 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6007999999999998e-05 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704487+0000", 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0505e-05 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704496+0000", 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.7709999999999995e-06 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704536+0000", 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0244000000000003e-05 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.704559+0000", 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2306999999999998e-05 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_repop(client.45605.0:91732 9.4 e249/241 9:2bbf588a:::benchmark_data_smithi125_199533_object91731:head v 249'5813, mlcod=249'5811)", 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:45.705871+0000", 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 0.00070853699999999997, 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001226186, 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "started", 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705871+0000", 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705871+0000", 2023-10-09T06:40:45.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705871+0000", 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705960+0000", 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9116000000000005e-05 2023-10-09T06:40:45.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705969+0000", 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1220000000000001e-06 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.705975+0000", 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4459999999999998e-06 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.706030+0000", 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4979999999999999e-05 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:45.706048+0000", 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7609e-05 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.767 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:40:45.768 INFO:teuthology.orchestra.run.smithi125.stdout: ], 2023-10-09T06:40:45.768 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 9 2023-10-09T06:40:45.768 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:40:45.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:45.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:45.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:45.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:45.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:45.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 91954 91938 77.6408 46.625 0.0123534 0.012874 2023-10-09T06:40:46.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:46.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:46.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:46.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:46.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:46 smithi155 bash[59495]: cluster 2023-10-09T06:40:45.273475+0000 mgr.y (mgr.25723) 1084 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:40:46.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:46.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.589 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:46.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:46 smithi125 bash[77299]: cluster 2023-10-09T06:40:45.273475+0000 mgr.y (mgr.25723) 1084 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:40:46.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:46 smithi125 bash[80002]: cluster 2023-10-09T06:40:45.273475+0000 mgr.y (mgr.25723) 1084 : cluster 0 pgmap v1141: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 920 op/s 2023-10-09T06:40:46.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:46.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:46.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:46.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:46.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 92811 92795 77.3197 53.5625 0.010302 0.0129268 2023-10-09T06:40:46.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:46.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:47.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:47.224 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:47.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:47.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:47.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:47.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:47.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:47.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:47.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:47.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:47.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 16 93615 93599 76.9635 50.25 0.0189715 0.0129872 2023-10-09T06:40:47.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:48.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:48.157 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:48.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:48.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:48 smithi125 bash[77299]: cluster 2023-10-09T06:40:47.273944+0000 mgr.y (mgr.25723) 1085 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T06:40:48.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:48 smithi125 bash[80002]: cluster 2023-10-09T06:40:47.273944+0000 mgr.y (mgr.25723) 1085 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T06:40:48.403 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:48.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:48 smithi155 bash[59495]: cluster 2023-10-09T06:40:47.273944+0000 mgr.y (mgr.25723) 1085 : cluster 0 pgmap v1142: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-10-09T06:40:48.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:48.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:48.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:48.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 94494 94478 76.6774 54.9375 0.0133711 0.0130364 2023-10-09T06:40:48.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:48.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:48.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:48.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:49.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:49.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:49.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:49.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:49.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:49.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:49.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:49 smithi125 bash[77299]: audit 2023-10-09T06:40:48.600039+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:49.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:49 smithi125 bash[80002]: audit 2023-10-09T06:40:48.600039+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:49 smithi155 bash[59495]: audit 2023-10-09T06:40:48.600039+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:49.665 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:49.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:49.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:49.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:49.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:49.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:49.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:49.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 95098 95082 76.1783 37.75 0.0174301 0.0131201 2023-10-09T06:40:50.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:50.051 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:50.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:50.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:50.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:50.314 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:50.410 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:50 smithi125 bash[77299]: cluster 2023-10-09T06:40:49.274834+0000 mgr.y (mgr.25723) 1086 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T06:40:50.410 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:50 smithi125 bash[80002]: cluster 2023-10-09T06:40:49.274834+0000 mgr.y (mgr.25723) 1086 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T06:40:50.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:50.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:50.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:50.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:50.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:50 smithi155 bash[59495]: cluster 2023-10-09T06:40:49.274834+0000 mgr.y (mgr.25723) 1086 : cluster 0 pgmap v1143: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T06:40:50.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:50.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:50.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:50.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:50.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 95904 95888 75.8516 50.375 0.00846857 0.0131778 2023-10-09T06:40:50.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:50.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:50.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:51.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:51.125 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:51.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:40:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:40:51] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T06:40:51.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:51.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:51.364 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:51.399 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:51.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:51.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:51.638 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:51.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:51.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:51.800 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:51.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:51.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:40:51.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:51.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:51.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:51.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:51.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:40:51.943032+0000 min lat: 0.00126636 max lat: 0.383876 avg lat: 0.0132294 2023-10-09T06:40:51.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:51.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 96743 96727 75.5589 52.4375 0.0178104 0.0132294 2023-10-09T06:40:52.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:52.072 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:52.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:52.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:52.346 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:52.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:52.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:52.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:52.597 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:52 smithi125 bash[77299]: cluster 2023-10-09T06:40:51.275480+0000 mgr.y (mgr.25723) 1087 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:52.597 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:52 smithi125 bash[80002]: cluster 2023-10-09T06:40:51.275480+0000 mgr.y (mgr.25723) 1087 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:52.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:52 smithi155 bash[59495]: cluster 2023-10-09T06:40:51.275480+0000 mgr.y (mgr.25723) 1087 : cluster 0 pgmap v1144: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:40:52.719 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:52.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.879 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:52.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:52.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 97435 97419 75.16 43.25 0.014233 0.0132991 2023-10-09T06:40:52.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:53.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:53.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:53.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:53.268 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:53.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:53.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:53.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:53.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:53.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:53.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:53.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:53.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:53.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 98289 98273 74.8943 53.375 0.0369292 0.0133463 2023-10-09T06:40:53.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:53.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:53.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:54.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:54.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:54.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:54.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:54.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:54.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:54.374 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:54.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:54.485 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:54 smithi125 bash[77299]: cluster 2023-10-09T06:40:53.276002+0000 mgr.y (mgr.25723) 1088 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T06:40:54.485 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:54 smithi125 bash[80002]: cluster 2023-10-09T06:40:53.276002+0000 mgr.y (mgr.25723) 1088 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T06:40:54.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:54.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:54.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:54 smithi155 bash[59495]: cluster 2023-10-09T06:40:53.276002+0000 mgr.y (mgr.25723) 1088 : cluster 0 pgmap v1145: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T06:40:54.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:54.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:54.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:54.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:54.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:54.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 99109 99093 74.6093 51.25 0.0216586 0.013397 2023-10-09T06:40:55.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:55.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:55.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:55.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:55.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:55.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:55.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:55.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:55.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:55.434 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:55.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:55.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:55.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:55.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:55.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:55.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:55.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:55.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:55.949 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 99967 99951 74.3595 53.625 0.00779361 0.0134418 2023-10-09T06:40:55.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:55.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:56.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:56.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:56.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:56.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:56.241 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:56.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:56.370 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:56.474 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:56.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:56.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:56.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:56 smithi125 bash[77299]: cluster 2023-10-09T06:40:55.276660+0000 mgr.y (mgr.25723) 1089 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:40:56.624 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:56 smithi125 bash[80002]: cluster 2023-10-09T06:40:55.276660+0000 mgr.y (mgr.25723) 1089 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:40:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:56 smithi155 bash[59495]: cluster 2023-10-09T06:40:55.276660+0000 mgr.y (mgr.25723) 1089 : cluster 0 pgmap v1146: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-09T06:40:56.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:56.677 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:56.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:56.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:56.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:56.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:56.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:56.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:56.948 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 100776 100760 74.0795 50.5625 0.0149119 0.013493 2023-10-09T06:40:57.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:57.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:57.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:57.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:57.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:57.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:57.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:57.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:57.654 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:57.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:57.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:57.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:57.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:57.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 101547 101531 73.7765 48.1875 0.00600929 0.0135476 2023-10-09T06:40:57.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:57.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:58.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:58.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:58.199 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:58.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:58.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:58.386 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T06:40:58.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:58.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:58.541 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[80002]: cluster 2023-10-09T06:40:57.277046+0000 mgr.y (mgr.25723) 1090 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T06:40:58.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[80002]: audit 2023-10-09T06:40:57.852830+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.125:0/3384883005' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.542 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[80002]: audit 2023-10-09T06:40:57.854760+0000 mon.a (mon.0) 1332 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[77299]: cluster 2023-10-09T06:40:57.277046+0000 mgr.y (mgr.25723) 1090 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T06:40:58.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[77299]: audit 2023-10-09T06:40:57.852830+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.125:0/3384883005' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:58 smithi125 bash[77299]: audit 2023-10-09T06:40:57.854760+0000 mon.a (mon.0) 1332 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:58.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:58 smithi155 bash[59495]: cluster 2023-10-09T06:40:57.277046+0000 mgr.y (mgr.25723) 1090 : cluster 0 pgmap v1147: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T06:40:58.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:58 smithi155 bash[59495]: audit 2023-10-09T06:40:57.852830+0000 mon.c (mon.2) 101 : audit 1 from='client.? 172.21.15.125:0/3384883005' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:58 smithi155 bash[59495]: audit 2023-10-09T06:40:57.854760+0000 mon.a (mon.0) 1332 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:40:58.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:58.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:58.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:58.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:58.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:58.951 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 102357 102341 73.5103 50.625 0.00737017 0.0135978 2023-10-09T06:40:58.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:59.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:59.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:59.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:59.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:59.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:59.388 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:59.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: audit 2023-10-09T06:40:58.370700+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:40:59.391 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.370787+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.371571+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.380829+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.387694+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.389582+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.464469+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.392 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:40:59 smithi155 bash[59495]: cluster 2023-10-09T06:40:58.466464+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:59.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:59.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:59.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:59.570 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:40:59.624 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:40:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: audit 2023-10-09T06:40:58.370700+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:40:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.370787+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:40:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.371571+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.380829+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.387694+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.389582+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.464469+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[77299]: cluster 2023-10-09T06:40:58.466464+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: audit 2023-10-09T06:40:58.370700+0000 mon.a (mon.0) 1333 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.370787+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.371571+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.380829+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.387694+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.389582+0000 osd.4 (osd.4) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.464469+0000 osd.5 (osd.5) 193 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:40:59 smithi125 bash[80002]: cluster 2023-10-09T06:40:58.466464+0000 osd.5 (osd.5) 194 : cluster 3 failed to encode map e250 with expected crc 2023-10-09T06:40:59.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:59.678 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:40:59.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:59.718 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:40:59.754 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:40:59.787 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:40:59.798 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:59.828 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:40:59.833 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:59.859 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:40:59.865 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.865 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.865 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.865 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.866 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:40:59.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:59.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 15 103044 103029 73.1631 43 0.0121807 0.0136621 2023-10-09T06:40:59.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:59.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:40:59.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:00.063 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:00.085 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:00.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:00.198 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:00.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:00.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:00.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:00.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:00.409 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:41:00.458 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[77299]: cluster 2023-10-09T06:40:59.277447+0000 mgr.y (mgr.25723) 1091 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s 2023-10-09T06:41:00.458 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[77299]: audit 2023-10-09T06:40:59.530103+0000 mon.a (mon.0) 1335 : audit 0 from='client.? 172.21.15.125:0/3403023056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:00.458 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[77299]: audit 2023-10-09T06:40:59.982657+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.125:0/3808387361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.458 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[77299]: audit 2023-10-09T06:40:59.983978+0000 mon.a (mon.0) 1336 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[80002]: cluster 2023-10-09T06:40:59.277447+0000 mgr.y (mgr.25723) 1091 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s 2023-10-09T06:41:00.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[80002]: audit 2023-10-09T06:40:59.530103+0000 mon.a (mon.0) 1335 : audit 0 from='client.? 172.21.15.125:0/3403023056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:00.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[80002]: audit 2023-10-09T06:40:59.982657+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.125:0/3808387361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:00 smithi125 bash[80002]: audit 2023-10-09T06:40:59.983978+0000 mon.a (mon.0) 1336 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:00.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:00.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:00.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:00.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:00.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:00.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:00.742 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:00 smithi155 bash[59495]: cluster 2023-10-09T06:40:59.277447+0000 mgr.y (mgr.25723) 1091 : cluster 0 pgmap v1149: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s 2023-10-09T06:41:00.742 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:00 smithi155 bash[59495]: audit 2023-10-09T06:40:59.530103+0000 mon.a (mon.0) 1335 : audit 0 from='client.? 172.21.15.125:0/3403023056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:00.742 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:00 smithi155 bash[59495]: audit 2023-10-09T06:40:59.982657+0000 mon.c (mon.2) 102 : audit 1 from='client.? 172.21.15.125:0/3808387361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.742 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:00 smithi155 bash[59495]: audit 2023-10-09T06:40:59.983978+0000 mon.a (mon.0) 1336 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:00.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:00.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:00.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:00.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:00.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 103746 103730 72.8333 43.8125 0.00563845 0.0137245 2023-10-09T06:41:00.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:01.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:01.378 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:01] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-10-09T06:41:01.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:01.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.572 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: cluster 2023-10-09T06:41:00.384554+0000 mon.a (mon.0) 1337 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: audit 2023-10-09T06:41:00.393325+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: cluster 2023-10-09T06:41:00.393578+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: cluster 2023-10-09T06:41:00.407821+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: cluster 2023-10-09T06:41:00.407821+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[77299]: cluster 2023-10-09T06:41:00.756890+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: cluster 2023-10-09T06:41:00.384554+0000 mon.a (mon.0) 1337 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: audit 2023-10-09T06:41:00.393325+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: cluster 2023-10-09T06:41:00.393578+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:41:01.573 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: cluster 2023-10-09T06:41:00.407821+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: cluster 2023-10-09T06:41:00.407821+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:01 smithi125 bash[80002]: cluster 2023-10-09T06:41:00.756890+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:01.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:01.703 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: cluster 2023-10-09T06:41:00.384554+0000 mon.a (mon.0) 1337 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:01.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: audit 2023-10-09T06:41:00.393325+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:01.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: cluster 2023-10-09T06:41:00.393578+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:41:01.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: cluster 2023-10-09T06:41:00.407821+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: cluster 2023-10-09T06:41:00.407821+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:01 smithi155 bash[59495]: cluster 2023-10-09T06:41:00.756890+0000 osd.5 (osd.5) 195 : cluster 3 failed to encode map e251 with expected crc 2023-10-09T06:41:01.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:01.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:01.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:01.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 14 104528 104514 72.5684 49 0.00397834 0.0137696 2023-10-09T06:41:01.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:01.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:02.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:02.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:02.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:02.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:02.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:02.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:02.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:02.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:02.554 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:02 smithi125 bash[77299]: cluster 2023-10-09T06:41:01.277927+0000 mgr.y (mgr.25723) 1092 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T06:41:02.554 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:02 smithi125 bash[80002]: cluster 2023-10-09T06:41:01.277927+0000 mgr.y (mgr.25723) 1092 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T06:41:02.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:02.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:02.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:02 smithi155 bash[59495]: cluster 2023-10-09T06:41:01.277927+0000 mgr.y (mgr.25723) 1092 : cluster 0 pgmap v1151: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-09T06:41:02.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:02.740 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:02.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:02.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:02.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:02.952 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.039 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 104528 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 72.5575 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 27.6919 2023-10-09T06:41:02.953 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 169 2023-10-09T06:41:02.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 37.75 2023-10-09T06:41:02.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1160 2023-10-09T06:41:02.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 443.071 2023-10-09T06:41:02.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2704 2023-10-09T06:41:02.954 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 604 2023-10-09T06:41:02.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0137768 2023-10-09T06:41:02.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.014152 2023-10-09T06:41:02.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 0.383876 2023-10-09T06:41:02.955 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00126636 2023-10-09T06:41:02.964 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:41:02.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:03.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:03.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:03.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:03.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:03.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:03.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:03.397 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:03.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:03.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:03.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:03.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:03.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:03.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:03.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:03.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:03.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:03.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:03.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:04.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:04.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:04.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:04.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:04.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:04.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:04 smithi155 bash[59495]: cluster 2023-10-09T06:41:03.278501+0000 mgr.y (mgr.25723) 1093 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s 2023-10-09T06:41:04.439 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:04 smithi155 bash[59495]: audit 2023-10-09T06:41:03.600291+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:04.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:04.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:04.568 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:04 smithi125 bash[80002]: cluster 2023-10-09T06:41:03.278501+0000 mgr.y (mgr.25723) 1093 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s 2023-10-09T06:41:04.568 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:04 smithi125 bash[80002]: audit 2023-10-09T06:41:03.600291+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:04.568 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:04 smithi125 bash[77299]: cluster 2023-10-09T06:41:03.278501+0000 mgr.y (mgr.25723) 1093 : cluster 0 pgmap v1152: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s 2023-10-09T06:41:04.569 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:04 smithi125 bash[77299]: audit 2023-10-09T06:41:03.600291+0000 mon.a (mon.0) 1340 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:04.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:04.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:04.717 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:04.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:04.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:04.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:04.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:05.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:05.230 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:05.264 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:05.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:05.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:05.505 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.551 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:05.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:05.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:05.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:05.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:05.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:05.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:41:05.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:05.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:06.066 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:06.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:06.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:06.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:06.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:06 smithi125 bash[77299]: cluster 2023-10-09T06:41:05.279187+0000 mgr.y (mgr.25723) 1094 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 46 MiB/s wr, 835 op/s 2023-10-09T06:41:06.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:06 smithi125 bash[80002]: cluster 2023-10-09T06:41:05.279187+0000 mgr.y (mgr.25723) 1094 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 46 MiB/s wr, 835 op/s 2023-10-09T06:41:06.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:06.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:06.614 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:06 smithi155 bash[59495]: cluster 2023-10-09T06:41:05.279187+0000 mgr.y (mgr.25723) 1094 : cluster 0 pgmap v1153: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 46 MiB/s wr, 835 op/s 2023-10-09T06:41:06.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:06.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.768 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:06.790 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:06.791 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:06.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:06.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:06.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:06.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:41:07.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:07.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:07.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:07.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:07.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:07.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:07.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:07.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.721 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:41:07.721 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:41:07.722 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 7, 6] 2023-10-09T06:41:07.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T06:41:07.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:07.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:07.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:07.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:07.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:08.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:08.107 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:08.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:08.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:41:08.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:08.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:08.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:08.417 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:08.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:08.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:08 smithi155 bash[59495]: cluster 2023-10-09T06:41:07.281749+0000 mgr.y (mgr.25723) 1095 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 114 B/s rd, 33 MiB/s wr, 650 op/s 2023-10-09T06:41:08.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:08.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:08.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:08.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:08 smithi125 bash[77299]: cluster 2023-10-09T06:41:07.281749+0000 mgr.y (mgr.25723) 1095 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 114 B/s rd, 33 MiB/s wr, 650 op/s 2023-10-09T06:41:08.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:08 smithi125 bash[80002]: cluster 2023-10-09T06:41:07.281749+0000 mgr.y (mgr.25723) 1095 : cluster 0 pgmap v1154: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 114 B/s rd, 33 MiB/s wr, 650 op/s 2023-10-09T06:41:08.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:08.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:08.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:08.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:08.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:08.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:09.069 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:09.099 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:09.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:09.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.236 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:09.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:09.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:09.360 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:09.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:09.496 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:09.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:09.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:09.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:09.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:09.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:09.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:09.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:09.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:09.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:10.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:10.038 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:10 smithi155 bash[59495]: cluster 2023-10-09T06:41:09.282347+0000 mgr.y (mgr.25723) 1096 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 709 op/s 2023-10-09T06:41:10.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:10.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:10 smithi125 bash[77299]: cluster 2023-10-09T06:41:09.282347+0000 mgr.y (mgr.25723) 1096 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 709 op/s 2023-10-09T06:41:10.140 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:10 smithi125 bash[80002]: cluster 2023-10-09T06:41:09.282347+0000 mgr.y (mgr.25723) 1096 : cluster 0 pgmap v1155: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 102 B/s rd, 29 MiB/s wr, 709 op/s 2023-10-09T06:41:10.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:10.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:10.394 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:10.419 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:10.526 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:10.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:10.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:10.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:10.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:10.937 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:10.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:11.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:11.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:11.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:11.256 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:11.360 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:11.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:11.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:11] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-10-09T06:41:11.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:11.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:11.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:11.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:11.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:11.731 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:11.732 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:11.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:11.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:11.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:11.960 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:11.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:12.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:12.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:12.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:12.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:12.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:12.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:12.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:12.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:12.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:12.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:12 smithi125 bash[77299]: cluster 2023-10-09T06:41:11.283195+0000 mgr.y (mgr.25723) 1097 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 94 B/s rd, 14 MiB/s wr, 648 op/s 2023-10-09T06:41:12.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:12 smithi125 bash[80002]: cluster 2023-10-09T06:41:11.283195+0000 mgr.y (mgr.25723) 1097 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 94 B/s rd, 14 MiB/s wr, 648 op/s 2023-10-09T06:41:12.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:12 smithi155 bash[59495]: cluster 2023-10-09T06:41:11.283195+0000 mgr.y (mgr.25723) 1097 : cluster 0 pgmap v1156: 89 pgs: 89 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 94 B/s rd, 14 MiB/s wr, 648 op/s 2023-10-09T06:41:12.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:12.752 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:12.758 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:12.758 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:12.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:12.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:12.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:12.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:13.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:13.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:13.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:13.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:13.294 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:13.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:13.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:13.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:13.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:13.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:13.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:13.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:13.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:13.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:13.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:13.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:13.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:14.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:14.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:14.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:14.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:14.347 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:14 smithi125 bash[77299]: cluster 2023-10-09T06:41:13.286029+0000 mgr.y (mgr.25723) 1098 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2023-10-09T06:41:14.347 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:14 smithi125 bash[80002]: cluster 2023-10-09T06:41:13.286029+0000 mgr.y (mgr.25723) 1098 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2023-10-09T06:41:14.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:14.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:14.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:14.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:14 smithi155 bash[59495]: cluster 2023-10-09T06:41:13.286029+0000 mgr.y (mgr.25723) 1098 : cluster 0 pgmap v1157: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 12 MiB/s wr, 729 op/s 2023-10-09T06:41:14.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:14.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:14.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:14.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:14.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:14.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:14.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:14.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:14.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:15.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:15.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:15.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:15.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:15.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:15.289 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:15.321 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:15.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:15.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:15.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:15.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:15.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:15.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:15.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:15.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:15.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:15.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:15.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:16.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:16.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:16.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:16.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:16.291 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:16.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:16 smithi155 bash[59495]: cluster 2023-10-09T06:41:15.286690+0000 mgr.y (mgr.25723) 1099 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 5.5 MiB/s wr, 798 op/s 2023-10-09T06:41:16.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:16.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:16.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:16.544 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:16 smithi125 bash[77299]: cluster 2023-10-09T06:41:15.286690+0000 mgr.y (mgr.25723) 1099 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 5.5 MiB/s wr, 798 op/s 2023-10-09T06:41:16.544 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:16 smithi125 bash[80002]: cluster 2023-10-09T06:41:15.286690+0000 mgr.y (mgr.25723) 1099 : cluster 0 pgmap v1158: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 85 B/s rd, 5.5 MiB/s wr, 798 op/s 2023-10-09T06:41:16.551 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:16.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:16.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:16.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:16.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:16.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:16.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:16.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:17.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:17.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:17.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:17.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:17.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:17.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:17.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:17.379 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:17.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:17.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:17.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:17.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:17.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:17.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:17.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:17.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:17.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:17.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:17.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:18.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:18.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:18.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:18.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:18.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:18.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:18.352 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:18 smithi125 bash[77299]: cluster 2023-10-09T06:41:17.287463+0000 mgr.y (mgr.25723) 1100 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 219 KiB/s wr, 694 op/s 2023-10-09T06:41:18.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:18 smithi125 bash[80002]: cluster 2023-10-09T06:41:17.287463+0000 mgr.y (mgr.25723) 1100 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 219 KiB/s wr, 694 op/s 2023-10-09T06:41:18.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:18.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:18.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.574 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:18 smithi155 bash[59495]: cluster 2023-10-09T06:41:17.287463+0000 mgr.y (mgr.25723) 1100 : cluster 0 pgmap v1159: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 219 KiB/s wr, 694 op/s 2023-10-09T06:41:18.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:18.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:18.726 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:18.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:18.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:18.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:19.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:19.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:19.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:19.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:19.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:19.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:19.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:19.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:19.514 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:19.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:19.627 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:19 smithi155 bash[59495]: audit 2023-10-09T06:41:18.601484+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:19.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:19.747 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:19.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:19.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:19.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:19 smithi125 bash[80002]: audit 2023-10-09T06:41:18.601484+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:19 smithi125 bash[77299]: audit 2023-10-09T06:41:18.601484+0000 mon.a (mon.0) 1341 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:19.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:19.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:20.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:20.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:20.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:20.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:20.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:20.354 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:20 smithi155 bash[59495]: cluster 2023-10-09T06:41:19.288026+0000 mgr.y (mgr.25723) 1101 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:41:20.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:20.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:20.581 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:20 smithi125 bash[77299]: cluster 2023-10-09T06:41:19.288026+0000 mgr.y (mgr.25723) 1101 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:41:20.582 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:20 smithi125 bash[80002]: cluster 2023-10-09T06:41:19.288026+0000 mgr.y (mgr.25723) 1101 : cluster 0 pgmap v1160: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 760 op/s 2023-10-09T06:41:20.601 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:20.705 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:20.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:20.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:20.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:20.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.012 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:21.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:21.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:21.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:21.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:21.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:21.526 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:21] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-10-09T06:41:21.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:21.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:21.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:21.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:21.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:21.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:21.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:22.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:22.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:22.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:22.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:22.333 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:22.357 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:22 smithi125 bash[77299]: cluster 2023-10-09T06:41:21.288675+0000 mgr.y (mgr.25723) 1102 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T06:41:22.357 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:22.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:22.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:22.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:22.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:22.628 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:22 smithi125 bash[80002]: cluster 2023-10-09T06:41:21.288675+0000 mgr.y (mgr.25723) 1102 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T06:41:22.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:22 smithi155 bash[59495]: cluster 2023-10-09T06:41:21.288675+0000 mgr.y (mgr.25723) 1102 : cluster 0 pgmap v1161: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 822 op/s 2023-10-09T06:41:22.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:22.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:22.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:22.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:22.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:22.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:23.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:23.148 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:23.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:23.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:23.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:23.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:23.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:23.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:23.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:23.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:23.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:23.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:23.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:23.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:24.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:24.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:24.154 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:24.178 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:24.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:24.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:24.364 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:24 smithi155 bash[59495]: cluster 2023-10-09T06:41:23.289198+0000 mgr.y (mgr.25723) 1103 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T06:41:24.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:24.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:24.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:24.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:24.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:24 smithi125 bash[77299]: cluster 2023-10-09T06:41:23.289198+0000 mgr.y (mgr.25723) 1103 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T06:41:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:24 smithi125 bash[80002]: cluster 2023-10-09T06:41:23.289198+0000 mgr.y (mgr.25723) 1103 : cluster 0 pgmap v1162: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 773 op/s 2023-10-09T06:41:24.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:24.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:24.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:24.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:24.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:24.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:24.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:25.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:25.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:25.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:25.250 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:25.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:25.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:25.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:25.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:25.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:25.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:25.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:25.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:25.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:25.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:25.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:26.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:26.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:26.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:26.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:26.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:26.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:26.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:26.367 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:26 smithi155 bash[59495]: cluster 2023-10-09T06:41:25.289812+0000 mgr.y (mgr.25723) 1104 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T06:41:26.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:26.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:26.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:26.585 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:26 smithi125 bash[77299]: cluster 2023-10-09T06:41:25.289812+0000 mgr.y (mgr.25723) 1104 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T06:41:26.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:26 smithi125 bash[80002]: cluster 2023-10-09T06:41:25.289812+0000 mgr.y (mgr.25723) 1104 : cluster 0 pgmap v1163: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T06:41:26.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:26.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:26.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:26.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:26.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:26.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:26.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:27.004 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:27.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:27.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:27.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:27.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:27.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:27.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:27.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:27.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:27.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:27.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:27.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:28.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:28.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:28.174 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:28.205 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:28.240 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:41:28.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:28.311 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:41:28.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:28.368 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:41:28.428 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:41:28.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:28.469 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:41:28.478 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:28 smithi155 bash[59495]: cluster 2023-10-09T06:41:27.290307+0000 mgr.y (mgr.25723) 1105 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T06:41:28.478 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:28 smithi155 bash[59495]: audit 2023-10-09T06:41:28.180299+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.125:0/3588419948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:28.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:28.523 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:41:28.562 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:41:28.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:28.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:28 smithi125 bash[77299]: cluster 2023-10-09T06:41:27.290307+0000 mgr.y (mgr.25723) 1105 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T06:41:28.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:28 smithi125 bash[77299]: audit 2023-10-09T06:41:28.180299+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.125:0/3588419948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:28.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:28 smithi125 bash[80002]: cluster 2023-10-09T06:41:27.290307+0000 mgr.y (mgr.25723) 1105 : cluster 0 pgmap v1164: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 667 op/s 2023-10-09T06:41:28.603 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:28 smithi125 bash[80002]: audit 2023-10-09T06:41:28.180299+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.125:0/3588419948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:28.609 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:41:28.620 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.621 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:41:28.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:28.631 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:41:28.631 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:41:28.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18138 8.b 8:de1a6651:::benchmark_data_smithi125_176369_object18137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 444.46481550099998, 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.347658947, 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.632 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18138 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157945+0000", 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157943+0000", 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:28.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157969+0000", 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6078999999999999e-05 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157986+0000", 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7246999999999999e-05 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.157995+0000", 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6240000000000008e-06 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158073+0000", 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8320999999999998e-05 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158244+0000", 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017047899999999999 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.158441+0000", 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000196925 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.159608+0000", 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011669180000000001 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.161628+0000", 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020198740000000001 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.638 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505490+0000", 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34386208600000001 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505553+0000", 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3676000000000005e-05 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.505604+0000", 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0297999999999999e-05 2023-10-09T06:41:28.639 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.640 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.640 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.640 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18162 8.0 8:07fcf932:::benchmark_data_smithi125_176369_object18161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:41:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:28.642 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:41:28.642 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 444.45223485999998, 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33879309499999999, 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18162 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170525+0000", 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170523+0000", 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170552+0000", 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8835000000000001e-05 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170566+0000", 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.452e-05 2023-10-09T06:41:28.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170575+0000", 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0750000000000004e-06 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170626+0000", 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.092e-05 2023-10-09T06:41:28.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170820+0000", 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000193764 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.170967+0000", 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000147372 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.647 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.172088+0000", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001120881 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.172269+0000", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018081400000000001 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509161+0000", 2023-10-09T06:41:28.648 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.336891579 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509250+0000", 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.9395999999999996e-05 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.509318+0000", 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.8211000000000001e-05 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.649 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:18175 8.f 8:ff073d6a:::benchmark_data_smithi125_176369_object18174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 444.44272535599998, 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33486070699999998, 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.650 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 18175 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:41:28.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180035+0000", 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180032+0000", 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999976 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180068+0000", 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5939999999999998e-05 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180092+0000", 2023-10-09T06:41:28.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3649e-05 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180105+0000", 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3287e-05 2023-10-09T06:41:28.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180162+0000", 2023-10-09T06:41:28.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6471999999999997e-05 2023-10-09T06:41:28.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180321+0000", 2023-10-09T06:41:28.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000159159 2023-10-09T06:41:28.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.180473+0000", 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000152292 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.181790+0000", 2023-10-09T06:41:28.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001316397 2023-10-09T06:41:28.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.181950+0000", 2023-10-09T06:41:28.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016058199999999999 2023-10-09T06:41:28.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514801+0000", 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33285042599999998 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514855+0000", 2023-10-09T06:41:28.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4542000000000001e-05 2023-10-09T06:41:28.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.662 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.662 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:04.514895+0000", 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0299999999999997e-05 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:87091 8.b 8:dfed4650:::benchmark_data_smithi125_176369_object87090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T06:41:28.663 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:59.652858+0000", 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.96990182799999, 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32475301499999998, 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.664 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87091 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652858+0000", 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652858+0000", 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652857+0000", 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652878+0000", 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0747e-05 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652891+0000", 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3244000000000001e-05 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652899+0000", 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2539999999999992e-06 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.652940+0000", 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0262000000000002e-05 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.661825+0000", 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0088850379999999996 2023-10-09T06:41:28.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.662291+0000", 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00046571699999999999 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.666506+0000", 2023-10-09T06:41:28.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042158129999999997 2023-10-09T06:41:28.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.669259+0000", 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027529239999999999 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.977526+0000", 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30826634800000002 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.977575+0000", 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8965999999999998e-05 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.977611+0000", 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6752000000000003e-05 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:87100 8.b 8:dd00f1de:::benchmark_data_smithi125_176369_object87099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e198)", 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:34:59.657290+0000", 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 388.96546967099999, 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32166700399999998, 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 87100 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.675 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657290+0000", 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657290+0000", 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657290+0000", 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657312+0000", 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2245000000000001e-05 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657331+0000", 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8943000000000001e-05 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.657339+0000", 2023-10-09T06:41:28.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9130000000000003e-06 2023-10-09T06:41:28.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.666532+0000", 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0091936980000000001 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.681 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.666682+0000", 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000149421 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.666789+0000", 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010675000000000001 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.669434+0000", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002645349 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.670394+0000", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00095994699999999995 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.978677+0000", 2023-10-09T06:41:28.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30828337300000003 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.978839+0000", 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016169599999999999 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:34:59.978957+0000", 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011848100000000001 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.684 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.685 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.685 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218069 8.13 8.ed4d5633 (undecoded) ondisk+write+known_if_redirected e219)", 2023-10-09T06:41:28.685 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:06.202482+0000", 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 202.420277672, 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.83877986000000004, 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218069 2023-10-09T06:41:28.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202482+0000", 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202482+0000", 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202481+0000", 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202487+0000", 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2809999999999999e-06 2023-10-09T06:41:28.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202507+0000", 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0395e-05 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202517+0000", 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6739999999999992e-06 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202803+0000", 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028592999999999999 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.202811+0000", 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0139999999999998e-06 2023-10-09T06:41:28.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041251+0000", 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.83843995800000004 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041262+0000", 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1684e-05 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.691 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218072 8.13 8.104d5f13 (undecoded) ondisk+write+known_if_redirected e219)", 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:06.206341+0000", 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 202.41641949800001, 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.834953995, 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:41:28.692 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218072 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206341+0000", 2023-10-09T06:41:28.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206341+0000", 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206339+0000", 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206347+0000", 2023-10-09T06:41:28.697 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.9910000000000002e-06 2023-10-09T06:41:28.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.697 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.697 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206365+0000", 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7997999999999999e-05 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206375+0000", 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0248000000000001e-05 2023-10-09T06:41:28.698 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206428+0000", 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3109000000000003e-05 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:41:28.699 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.206439+0000", 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0801e-05 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041286+0000", 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.83484759100000006 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041295+0000", 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1519999999999992e-06 2023-10-09T06:41:28.700 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218081 8.13 8.8d31a773 (undecoded) ondisk+write+known_if_redirected e219)", 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:06.217838+0000", 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 202.40492223199999, 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.82348364100000004, 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.701 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218081 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217838+0000", 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217838+0000", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217837+0000", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217843+0000", 2023-10-09T06:41:28.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7049999999999996e-06 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217852+0000", 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7499999999999998e-06 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217859+0000", 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0530000000000001e-06 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217929+0000", 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9358999999999998e-05 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.217940+0000", 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0763e-05 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041314+0000", 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.82337449699999998 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041322+0000", 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4320000000000002e-06 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.706 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218101 8.13 8.763699b3 (undecoded) ondisk+write+known_if_redirected e219)", 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:06.241704+0000", 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 202.38105614899999, 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79964416900000002, 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218101 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241704+0000", 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241704+0000", 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241702+0000", 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241712+0000", 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0251999999999999e-05 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241735+0000", 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2398000000000001e-05 2023-10-09T06:41:28.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241745+0000", 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0560000000000001e-05 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241832+0000", 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6261999999999995e-05 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:06.241841+0000", 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7820000000000005e-06 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041340+0000", 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.79949907099999995 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.041348+0000", 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6809999999999998e-06 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:1270 9.f 9:fb4381be:::benchmark_data_smithi125_199533_object1269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e247)", 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:39:32.380398+0000", 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.24236234200001, 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.34059959899999998, 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.712 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1270 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380398+0000", 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380398+0000", 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380397+0000", 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380424+0000", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6716999999999998e-05 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380433+0000", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3980000000000005e-06 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380440+0000", 2023-10-09T06:41:28.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0690000000000004e-06 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380481+0000", 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.1162000000000003e-05 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380611+0000", 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012929200000000001 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.380791+0000", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017981299999999999 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.382018+0000", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0012270099999999999 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.382036+0000", 2023-10-09T06:41:28.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8533e-05 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.720781+0000", 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33874533699999998 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.720900+0000", 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000118613 2023-10-09T06:41:28.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.720997+0000", 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7297e-05 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:1293 9.f 9:f476272e:::benchmark_data_smithi125_199533_object1292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e247)", 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:39:32.386822+0000", 2023-10-09T06:41:28.719 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.23593841500001, 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33501634899999999, 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1293 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386822+0000", 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386822+0000", 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386820+0000", 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:28.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386849+0000", 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8251000000000001e-05 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386861+0000", 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2165e-05 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386868+0000", 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6690000000000006e-06 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.386910+0000", 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2110999999999997e-05 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387030+0000", 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119911 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387165+0000", 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013430699999999999 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388717+0000", 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001552606 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388808+0000", 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0926000000000003e-05 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.721614+0000", 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.332805401 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.721742+0000", 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012791699999999999 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.721838+0000", 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6525000000000002e-05 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:1298 9.f 9:f32af297:::benchmark_data_smithi125_199533_object1297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e247)", 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:39:32.387648+0000", 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.235112015, 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33448767200000001, 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1298 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.727 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387648+0000", 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387648+0000", 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387647+0000", 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387677+0000", 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0222e-05 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387690+0000", 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2334000000000001e-05 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.387698+0000", 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0199999999999994e-06 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388191+0000", 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00049315800000000005 2023-10-09T06:41:28.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388327+0000", 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136534 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388474+0000", 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014672 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.388881+0000", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000406764 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.389590+0000", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070879099999999996 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.721950+0000", 2023-10-09T06:41:28.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33236046400000002 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.722052+0000", 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010207 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.722136+0000", 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.3551e-05 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.733 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:1307 9.f 9:f046e8a9:::benchmark_data_smithi125_199533_object1306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e247)", 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:39:32.391033+0000", 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 116.231726647, 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33137411900000002, 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.734 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 1307 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391033+0000", 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391033+0000", 2023-10-09T06:41:28.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391033+0000", 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391050+0000", 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6813e-05 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391058+0000", 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8300000000000002e-06 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391065+0000", 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0310000000000001e-06 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391114+0000", 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8627e-05 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391213+0000", 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.9160999999999995e-05 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391296+0000", 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.2993999999999999e-05 2023-10-09T06:41:28.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.391962+0000", 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000665323 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.392080+0000", 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011836999999999999 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.722229+0000", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33014900899999999 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.722327+0000", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8115999999999994e-05 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:39:32.722408+0000", 2023-10-09T06:41:28.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0544999999999995e-05 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94674 9.2 9:464599ad:::benchmark_data_smithi125_199533_object94673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.160875+0000", 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.461884867999999, 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32246808599999999, 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.741 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94674 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160875+0000", 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160875+0000", 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160874+0000", 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160931+0000", 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6366999999999998e-05 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160949+0000", 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8110000000000001e-05 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.160956+0000", 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4719999999999998e-06 2023-10-09T06:41:28.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171693+0000", 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010736802 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171792+0000", 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.8672000000000007e-05 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.745 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171882+0000", 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0011000000000003e-05 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.174695+0000", 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002813355 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.177264+0000", 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.002568655 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.483241+0000", 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30597672999999997 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.483298+0000", 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7793e-05 2023-10-09T06:41:28.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.483343+0000", 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4938999999999998e-05 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94685 9.f 9:f6c61815:::benchmark_data_smithi125_199533_object94684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.167720+0000", 2023-10-09T06:41:28.748 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.455040238000002, 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.328379317, 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94685 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167720+0000", 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167720+0000", 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167719+0000", 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167749+0000", 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0392e-05 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167759+0000", 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7529999999999996e-06 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.167767+0000", 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4220000000000003e-06 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.173867+0000", 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061001340000000001 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.174089+0000", 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022211600000000001 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.174217+0000", 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0001282 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183115+0000", 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0088983880000000001 2023-10-09T06:41:28.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183124+0000", 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0140000000000004e-06 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.495922+0000", 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31279782099999998 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.496058+0000", 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000136098 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.496099+0000", 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0920999999999997e-05 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94686 9.f 9:ff6e1f96:::benchmark_data_smithi125_199533_object94685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.170187+0000", 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.452573391999998, 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32613326100000001, 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94686 2023-10-09T06:41:28.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170187+0000", 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170187+0000", 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170185+0000", 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170233+0000", 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7647e-05 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170250+0000", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.6912e-05 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.170258+0000", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8860000000000004e-06 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.178467+0000", 2023-10-09T06:41:28.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0082091279999999996 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.178611+0000", 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014394500000000001 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.178742+0000", 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013152699999999999 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183159+0000", 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044167410000000001 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183165+0000", 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9859999999999999e-06 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.496235+0000", 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31307030699999999 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.496284+0000", 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8523999999999998e-05 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.496320+0000", 2023-10-09T06:41:28.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6254999999999998e-05 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94688 9.f 9:f83b6858:::benchmark_data_smithi125_199533_object94687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.171555+0000", 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.451204664000002, 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33120501600000002, 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.763 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94688 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171555+0000", 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171555+0000", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171554+0000", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171600+0000", 2023-10-09T06:41:28.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5427999999999999e-05 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171612+0000", 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2292e-05 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171619+0000", 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3420000000000002e-06 2023-10-09T06:41:28.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183187+0000", 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.011567622 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183342+0000", 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015487899999999999 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183499+0000", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000156976 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.189669+0000", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0061703169999999998 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192203+0000", 2023-10-09T06:41:28.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0025335620000000001 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502683+0000", 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.310480282 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502724+0000", 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0506999999999997e-05 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502760+0000", 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6860999999999998e-05 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94689 9.f 9:fe8e3ea0:::benchmark_data_smithi125_199533_object94688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.171834+0000", 2023-10-09T06:41:28.770 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.450925906999998, 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.33103890400000002, 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94689 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171834+0000", 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171834+0000", 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171833+0000", 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171852+0000", 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.8493000000000001e-05 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171861+0000", 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1530000000000003e-06 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.773 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.171868+0000", 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4960000000000001e-06 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.186519+0000", 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.014651114999999999 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.186631+0000", 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112117 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.186728+0000", 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6897000000000004e-05 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192188+0000", 2023-10-09T06:41:28.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0054600569999999999 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192210+0000", 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.245e-05 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502795+0000", 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31058508299999998 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502840+0000", 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4359e-05 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502873+0000", 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3457000000000002e-05 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.777 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94698 9.f 9:fdc2de0a:::benchmark_data_smithi125_199533_object94697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.181593+0000", 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.441166809000002, 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.32140533599999999, 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.778 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94698 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181593+0000", 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181593+0000", 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181592+0000", 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181609+0000", 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7076999999999999e-05 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181617+0000", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.199e-06 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.181621+0000", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.673e-06 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.189698+0000", 2023-10-09T06:41:28.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0080768360000000004 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.189842+0000", 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000143487 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.189999+0000", 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015717899999999999 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.194319+0000", 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043207330000000002 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.194349+0000", 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9984e-05 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502904+0000", 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30855403999999997 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502943+0000", 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9106000000000001e-05 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.502999+0000", 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.6022e-05 2023-10-09T06:41:28.784 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45605.0:94700 9.f 9:f5800ecd:::benchmark_data_smithi125_199533_object94699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:40:49.183622+0000", 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 39.439138317999998, 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.31952766500000002, 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:41:28.785 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45605", 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 94700 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183622+0000", 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:41:28.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183622+0000", 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183621+0000", 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.999999 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183652+0000", 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.1463e-05 2023-10-09T06:41:28.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183661+0000", 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.653e-06 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.183666+0000", 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3489999999999999e-06 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192230+0000", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.008563368 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192330+0000", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000100369 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.192435+0000", 2023-10-09T06:41:28.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000105345 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.194513+0000", 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020778770000000001 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.194635+0000", 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000121277 2023-10-09T06:41:28.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.503036+0000", 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.30840107700000002 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.503116+0000", 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.0696000000000001e-05 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:41:28.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:40:49.503149+0000", 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3148999999999997e-05 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:41:28.792 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:41:28.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:28.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:28.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:28.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:28.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:28.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:29.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.096 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:29.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:29.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:29.225 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:29.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:29.339 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: debug 2023-10-09T06:41:29.060+0000 7ff949fd9700 -1 mon.a@0(leader).osd e251 definitely_dead 0 2023-10-09T06:41:29.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:29.398 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.6. 2023-10-09T06:41:29.400 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:41:29.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:29.519 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:29.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:29.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:29.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:29 smithi155 bash[59495]: audit 2023-10-09T06:41:28.691136+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.125:0/3115976479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:29 smithi155 bash[59495]: audit 2023-10-09T06:41:28.692054+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:29 smithi155 bash[59495]: audit 2023-10-09T06:41:29.065047+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.125:0/4211686907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:29 smithi155 bash[59495]: audit 2023-10-09T06:41:29.066097+0000 mon.a (mon.0) 1343 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:29 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.066394+0000 mon.a (mon.0) 1344 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:41:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[80002]: audit 2023-10-09T06:41:28.691136+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.125:0/3115976479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[80002]: audit 2023-10-09T06:41:28.692054+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[80002]: audit 2023-10-09T06:41:29.065047+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.125:0/4211686907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[80002]: audit 2023-10-09T06:41:29.066097+0000 mon.a (mon.0) 1343 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.066394+0000 mon.a (mon.0) 1344 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: audit 2023-10-09T06:41:28.691136+0000 mon.c (mon.2) 103 : audit 1 from='client.? 172.21.15.125:0/3115976479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: audit 2023-10-09T06:41:28.692054+0000 mon.a (mon.0) 1342 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: audit 2023-10-09T06:41:29.065047+0000 mon.c (mon.2) 104 : audit 1 from='client.? 172.21.15.125:0/4211686907' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: audit 2023-10-09T06:41:29.066097+0000 mon.a (mon.0) 1343 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:41:29.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:29 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.066394+0000 mon.a (mon.0) 1344 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:41:29.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:29.754 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:29.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:29.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:29.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:29.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:30.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:30.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:30.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:30.192 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:30.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:30.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:30.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.291232+0000 mgr.y (mgr.25723) 1106 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.364288+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.381576+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: audit 2023-10-09T06:41:29.382084+0000 mon.a (mon.0) 1346 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: audit 2023-10-09T06:41:29.382154+0000 mon.a (mon.0) 1347 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.382203+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e252: 8 total, 8 up, 6 in 2023-10-09T06:41:30.445 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:30 smithi155 bash[59495]: cluster 2023-10-09T06:41:29.383620+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:30.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.291232+0000 mgr.y (mgr.25723) 1106 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T06:41:30.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.364288+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:30.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.381576+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: audit 2023-10-09T06:41:29.382084+0000 mon.a (mon.0) 1346 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:30.480 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: audit 2023-10-09T06:41:29.382154+0000 mon.a (mon.0) 1347 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:41:30.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.382203+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e252: 8 total, 8 up, 6 in 2023-10-09T06:41:30.481 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[77299]: cluster 2023-10-09T06:41:29.383620+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:30.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:30.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:30.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:30.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:30.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:30.860 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:30.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.291232+0000 mgr.y (mgr.25723) 1106 : cluster 0 pgmap v1165: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 723 op/s 2023-10-09T06:41:30.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.364288+0000 mon.a (mon.0) 1345 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:30.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.381576+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: audit 2023-10-09T06:41:29.382084+0000 mon.a (mon.0) 1346 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:30.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: audit 2023-10-09T06:41:29.382154+0000 mon.a (mon.0) 1347 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:41:30.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.382203+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e252: 8 total, 8 up, 6 in 2023-10-09T06:41:30.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:30 smithi125 bash[80002]: cluster 2023-10-09T06:41:29.383620+0000 osd.4 (osd.4) 194 : cluster 3 failed to encode map e252 with expected crc 2023-10-09T06:41:30.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:30.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:30.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:30.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:31.094 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:31.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:31.231 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:31.240 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:31] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.33.4" 2023-10-09T06:41:31.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:31.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:31.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:31.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.379713+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e253: 8 total, 8 up, 6 in 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.382431+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.384998+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.386649+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.386780+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.390841+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.391121+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.507 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.469485+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.470987+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.508 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[77299]: cluster 2023-10-09T06:41:30.472156+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.508 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.379713+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e253: 8 total, 8 up, 6 in 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.382431+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.384998+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.386649+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.386780+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.390841+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.391121+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.509 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.469485+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.470987+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.510 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:31 smithi125 bash[80002]: cluster 2023-10-09T06:41:30.472156+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:31.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.379713+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e253: 8 total, 8 up, 6 in 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.382431+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.384998+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.386649+0000 osd.4 (osd.4) 195 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.386780+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.390841+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.391121+0000 osd.4 (osd.4) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.469485+0000 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.470987+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:31 smithi155 bash[59495]: cluster 2023-10-09T06:41:30.472156+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e253 with expected crc 2023-10-09T06:41:31.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:31.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:31.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:31.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:31.902 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:31.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:31.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:31.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:32.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:32.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:32.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:32.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:32.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.316 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:32.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:32.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.291945+0000 mgr.y (mgr.25723) 1107 : cluster 0 pgmap v1168: 89 pgs: 18 peering, 71 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T06:41:32.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.376379+0000 mon.a (mon.0) 1350 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:41:32.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.388525+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e254: 8 total, 8 up, 6 in 2023-10-09T06:41:32.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.388917+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.434 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.388975+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.435 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.392173+0000 osd.3 (osd.3) 14 : cluster 0 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 251'7838 2023-10-09T06:41:32.435 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.392509+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.435 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.392560+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.435 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:32 smithi155 bash[59495]: cluster 2023-10-09T06:41:31.395937+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:32.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:32.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:32.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:32.692 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.291945+0000 mgr.y (mgr.25723) 1107 : cluster 0 pgmap v1168: 89 pgs: 18 peering, 71 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.376379+0000 mon.a (mon.0) 1350 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.388525+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e254: 8 total, 8 up, 6 in 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.388917+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.388975+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.392173+0000 osd.3 (osd.3) 14 : cluster 0 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 251'7838 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.392509+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.392560+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[77299]: cluster 2023-10-09T06:41:31.395937+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.693 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.291945+0000 mgr.y (mgr.25723) 1107 : cluster 0 pgmap v1168: 89 pgs: 18 peering, 71 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 738 op/s; 0 B/s, 1 objects/s recovering 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.376379+0000 mon.a (mon.0) 1350 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.388525+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e254: 8 total, 8 up, 6 in 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.388917+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.388975+0000 osd.4 (osd.4) 197 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.392173+0000 osd.3 (osd.3) 14 : cluster 0 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 251'7838 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.392509+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.392560+0000 osd.4 (osd.4) 198 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:32 smithi125 bash[80002]: cluster 2023-10-09T06:41:31.395937+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:32.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:32.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:32.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:32.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:33.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:41:33.070 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:33.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:33.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:33.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:33.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:33.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:33.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:33.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:33.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:33.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[77299]: cluster 2023-10-09T06:41:32.083792+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:33.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[77299]: cluster 2023-10-09T06:41:32.399183+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e255: 8 total, 8 up, 6 in 2023-10-09T06:41:33.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[77299]: cluster 2023-10-09T06:41:32.399892+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[77299]: cluster 2023-10-09T06:41:32.403029+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[77299]: cluster 2023-10-09T06:41:32.558839+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[80002]: cluster 2023-10-09T06:41:32.083792+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[80002]: cluster 2023-10-09T06:41:32.399183+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e255: 8 total, 8 up, 6 in 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[80002]: cluster 2023-10-09T06:41:32.399892+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[80002]: cluster 2023-10-09T06:41:32.403029+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:33 smithi125 bash[80002]: cluster 2023-10-09T06:41:32.558839+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:33 smithi155 bash[59495]: cluster 2023-10-09T06:41:32.083792+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e254 with expected crc 2023-10-09T06:41:33.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:33 smithi155 bash[59495]: cluster 2023-10-09T06:41:32.399183+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e255: 8 total, 8 up, 6 in 2023-10-09T06:41:33.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:33 smithi155 bash[59495]: cluster 2023-10-09T06:41:32.399892+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:33 smithi155 bash[59495]: cluster 2023-10-09T06:41:32.403029+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.641 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:33 smithi155 bash[59495]: cluster 2023-10-09T06:41:32.558839+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e255 with expected crc 2023-10-09T06:41:33.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:33.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:33.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:33.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:33.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:33.922 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:33.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:34.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:34.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:34.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:34.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:34.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:34.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:34.431 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.292612+0000 mgr.y (mgr.25723) 1108 : cluster 0 pgmap v1171: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 18 peering, 68 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 13058/251832 objects degraded (5.185%); 5055/251832 objects misplaced (2.007%); 3.1 MiB/s, 52 objects/s recovering 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.394829+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: Degraded data redundancy: 13058/251832 objects degraded (5.185%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.415227+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e256: 8 total, 8 up, 6 in 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.415858+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.419113+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.420574+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: cluster 2023-10-09T06:41:33.420686+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: audit 2023-10-09T06:41:33.606140+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:34.440 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:34 smithi155 bash[59495]: audit 2023-10-09T06:41:33.609983+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:34.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:34.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:34.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:41:34.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:34.630 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:34.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.292612+0000 mgr.y (mgr.25723) 1108 : cluster 0 pgmap v1171: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 18 peering, 68 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 13058/251832 objects degraded (5.185%); 5055/251832 objects misplaced (2.007%); 3.1 MiB/s, 52 objects/s recovering 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.394829+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: Degraded data redundancy: 13058/251832 objects degraded (5.185%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.415227+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e256: 8 total, 8 up, 6 in 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.415858+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.419113+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.420574+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: cluster 2023-10-09T06:41:33.420686+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: audit 2023-10-09T06:41:33.606140+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:34.748 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[77299]: audit 2023-10-09T06:41:33.609983+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.292612+0000 mgr.y (mgr.25723) 1108 : cluster 0 pgmap v1171: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 18 peering, 68 active+clean; 5.1 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 13058/251832 objects degraded (5.185%); 5055/251832 objects misplaced (2.007%); 3.1 MiB/s, 52 objects/s recovering 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.394829+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: Degraded data redundancy: 13058/251832 objects degraded (5.185%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.415227+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e256: 8 total, 8 up, 6 in 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.415858+0000 osd.4 (osd.4) 199 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.419113+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.420574+0000 osd.4 (osd.4) 200 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: cluster 2023-10-09T06:41:33.420686+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e256 with expected crc 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: audit 2023-10-09T06:41:33.606140+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:34.749 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:34 smithi125 bash[80002]: audit 2023-10-09T06:41:33.609983+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:34.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:34.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:34.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:34.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:35.017 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:35.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:35.178 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:35.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:35.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:35.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:35.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:35.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:35.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:35 smithi125 bash[77299]: cluster 2023-10-09T06:41:35.293381+0000 mgr.y (mgr.25723) 1109 : cluster 0 pgmap v1173: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 346 op/s; 13136/248976 objects degraded (5.276%); 10217/248976 objects misplaced (4.104%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:41:35.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:35 smithi125 bash[77299]: cluster 2023-10-09T06:41:35.407507+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:41:35.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:35 smithi125 bash[80002]: cluster 2023-10-09T06:41:35.293381+0000 mgr.y (mgr.25723) 1109 : cluster 0 pgmap v1173: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 346 op/s; 13136/248976 objects degraded (5.276%); 10217/248976 objects misplaced (4.104%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:41:35.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:35 smithi125 bash[80002]: cluster 2023-10-09T06:41:35.407507+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:41:35.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:35.730 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:35.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:35.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:35.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:35 smithi155 bash[59495]: cluster 2023-10-09T06:41:35.293381+0000 mgr.y (mgr.25723) 1109 : cluster 0 pgmap v1173: 89 pgs: 1 active+remapped+backfilling, 2 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 346 op/s; 13136/248976 objects degraded (5.276%); 10217/248976 objects misplaced (4.104%); 3.2 MiB/s, 55 objects/s recovering 2023-10-09T06:41:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:35 smithi155 bash[59495]: cluster 2023-10-09T06:41:35.407507+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:41:35.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:36.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:36.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:36.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:36.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:36.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:36.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:36.376 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.403 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:36.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:36.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:36.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:41:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:36.648 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:36.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:36.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:36.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:36.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:37.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:37.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:37.172 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:37.317 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.345 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:37.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:37.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:37.585 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:37.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:37.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:37.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:37.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:37.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:37.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:37.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:38.051 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:38.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.152 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:38.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.286 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:38.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:38.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:38 smithi155 bash[59495]: cluster 2023-10-09T06:41:37.293978+0000 mgr.y (mgr.25723) 1110 : cluster 0 pgmap v1174: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 13135/248043 objects degraded (5.295%); 10217/248043 objects misplaced (4.119%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T06:41:38.413 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:38.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:38.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:38.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:38 smithi125 bash[77299]: cluster 2023-10-09T06:41:37.293978+0000 mgr.y (mgr.25723) 1110 : cluster 0 pgmap v1174: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 13135/248043 objects degraded (5.295%); 10217/248043 objects misplaced (4.119%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T06:41:38.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:38 smithi125 bash[80002]: cluster 2023-10-09T06:41:37.293978+0000 mgr.y (mgr.25723) 1110 : cluster 0 pgmap v1174: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 13135/248043 objects degraded (5.295%); 10217/248043 objects misplaced (4.119%); 2.6 MiB/s, 45 objects/s recovering 2023-10-09T06:41:38.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:38.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:38.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:38.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:38.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:38.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:38.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:39.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:39.082 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:39.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:39.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:39.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:39.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:39.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:39.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:39 smithi125 bash[77299]: cluster 2023-10-09T06:41:39.025236+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 13135/248043 objects degraded (5.295%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:39.419 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:39.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:39.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:39.502 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:41:39.502 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:41:39.503 INFO:tasks.thrashosds.thrasher:Reweighting osd 1 to 0.1670432103403715 2023-10-09T06:41:39.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 0.1670432103403715 2023-10-09T06:41:39.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:39.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:39 smithi155 bash[59495]: cluster 2023-10-09T06:41:39.025236+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 13135/248043 objects degraded (5.295%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:39.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:39.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:39 smithi125 bash[80002]: cluster 2023-10-09T06:41:39.025236+0000 mon.a (mon.0) 1358 : cluster 3 Health check update: Degraded data redundancy: 13135/248043 objects degraded (5.295%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:39.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:39.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:39.848 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:39.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:39.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:39.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:40.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:40.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:40.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:40.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:40.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:40.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:40.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:40.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:40.437 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:40.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:40.545 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:40 smithi155 bash[59495]: cluster 2023-10-09T06:41:39.294536+0000 mgr.y (mgr.25723) 1111 : cluster 0 pgmap v1175: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 371 op/s; 13135/245268 objects degraded (5.355%); 9487/245268 objects misplaced (3.868%); 6.3 MiB/s, 104 objects/s recovering 2023-10-09T06:41:40.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:40.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:40.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:40.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:40 smithi125 bash[77299]: cluster 2023-10-09T06:41:39.294536+0000 mgr.y (mgr.25723) 1111 : cluster 0 pgmap v1175: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 371 op/s; 13135/245268 objects degraded (5.355%); 9487/245268 objects misplaced (3.868%); 6.3 MiB/s, 104 objects/s recovering 2023-10-09T06:41:40.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:40 smithi125 bash[80002]: cluster 2023-10-09T06:41:39.294536+0000 mgr.y (mgr.25723) 1111 : cluster 0 pgmap v1175: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 371 op/s; 13135/245268 objects degraded (5.355%); 9487/245268 objects misplaced (3.868%); 6.3 MiB/s, 104 objects/s recovering 2023-10-09T06:41:40.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:40.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:40.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:40.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:40.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:40.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:41.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:41.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:41.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:41.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:41.333 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:41.355 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:41.428 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:41] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-10-09T06:41:41.428 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:41 smithi125 bash[80002]: audit 2023-10-09T06:41:40.835104+0000 mon.a (mon.0) 1359 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:41.459 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:41.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:41.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:41.642 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:41 smithi155 bash[59495]: audit 2023-10-09T06:41:40.835104+0000 mon.a (mon.0) 1359 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:41.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:41.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:41 smithi125 bash[77299]: audit 2023-10-09T06:41:40.835104+0000 mon.a (mon.0) 1359 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:41.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:41.891 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:41.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:41.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:42.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:42.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:42.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:42.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:42.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:42.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:42.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:42.659 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:42 smithi155 bash[59495]: cluster 2023-10-09T06:41:41.295227+0000 mgr.y (mgr.25723) 1112 : cluster 0 pgmap v1176: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13227/240246 objects degraded (5.506%); 9318/240246 objects misplaced (3.879%); 6.1 MiB/s, 101 objects/s recovering 2023-10-09T06:41:42.660 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:42 smithi125 bash[77299]: cluster 2023-10-09T06:41:41.295227+0000 mgr.y (mgr.25723) 1112 : cluster 0 pgmap v1176: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13227/240246 objects degraded (5.506%); 9318/240246 objects misplaced (3.879%); 6.1 MiB/s, 101 objects/s recovering 2023-10-09T06:41:42.660 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:42 smithi125 bash[80002]: cluster 2023-10-09T06:41:41.295227+0000 mgr.y (mgr.25723) 1112 : cluster 0 pgmap v1176: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 13227/240246 objects degraded (5.506%); 9318/240246 objects misplaced (3.879%); 6.1 MiB/s, 101 objects/s recovering 2023-10-09T06:41:42.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:42.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:42.929 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:42.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:43.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:43.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:43.195 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:43.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:43.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:43.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:43.495 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:43.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:43.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:43.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:43.663 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:41:43.663 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:41:43.663 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.663 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45605.0:131811 9.1 e256/253 9:82bd0a8d:::benchmark_data_smithi125_199533_object27280:head v 256'8301, mlcod=256'8299)", 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:41:43.658651+0000", 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.002285666, 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023301110000000002, 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658651+0000", 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.664 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658651+0000", 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658649+0000", 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0492999999999999e-05 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:41:43.665 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658660+0000", 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6957e-05 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658677+0000", 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658685+0000", 2023-10-09T06:41:43.666 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4279999999999999e-05 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658769+0000", 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2292e-05 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:41:43.658821+0000", 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017084399999999999 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:41:43.667 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:41:43.668 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:41:43.668 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:41:43.668 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:41:43.668 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T06:41:43.668 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:41:43.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:43.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:43.896 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:43.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:44.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:44.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.220 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:44.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:44.349 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:44.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[77299]: cluster 2023-10-09T06:41:43.295888+0000 mgr.y (mgr.25723) 1113 : cluster 0 pgmap v1177: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13227/236436 objects degraded (5.594%); 8624/236436 objects misplaced (3.647%); 7.6 MiB/s, 124 objects/s recovering 2023-10-09T06:41:44.376 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[77299]: cluster 2023-10-09T06:41:44.026170+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 13227/236436 objects degraded (5.594%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:44.377 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[77299]: audit 2023-10-09T06:41:44.251489+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:44.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:44.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:44.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:44 smithi155 bash[59495]: cluster 2023-10-09T06:41:43.295888+0000 mgr.y (mgr.25723) 1113 : cluster 0 pgmap v1177: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13227/236436 objects degraded (5.594%); 8624/236436 objects misplaced (3.647%); 7.6 MiB/s, 124 objects/s recovering 2023-10-09T06:41:44.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:44 smithi155 bash[59495]: cluster 2023-10-09T06:41:44.026170+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 13227/236436 objects degraded (5.594%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:44.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:44 smithi155 bash[59495]: audit 2023-10-09T06:41:44.251489+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:44.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:44.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[80002]: cluster 2023-10-09T06:41:43.295888+0000 mgr.y (mgr.25723) 1113 : cluster 0 pgmap v1177: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 13227/236436 objects degraded (5.594%); 8624/236436 objects misplaced (3.647%); 7.6 MiB/s, 124 objects/s recovering 2023-10-09T06:41:44.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[80002]: cluster 2023-10-09T06:41:44.026170+0000 mon.a (mon.0) 1360 : cluster 3 Health check update: Degraded data redundancy: 13227/236436 objects degraded (5.594%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:44.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:44 smithi125 bash[80002]: audit 2023-10-09T06:41:44.251489+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:44.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:41:44.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:44.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:44.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:44.870 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:44.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:44.974 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:45.009 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:45.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:45.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:45.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:45.281 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:45.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:45.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:45.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:45.555 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:45.659 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:45.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:45.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:45.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:45.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:45.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:45.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:45.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:46.055 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:46.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:46.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:46.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:46.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:46.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:46.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:46.493 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:46.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:46.606 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:46 smithi155 bash[59495]: cluster 2023-10-09T06:41:45.296569+0000 mgr.y (mgr.25723) 1114 : cluster 0 pgmap v1178: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 13312/231627 objects degraded (5.747%); 8454/231627 objects misplaced (3.650%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T06:41:46.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:46.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:46 smithi125 bash[77299]: cluster 2023-10-09T06:41:45.296569+0000 mgr.y (mgr.25723) 1114 : cluster 0 pgmap v1178: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 13312/231627 objects degraded (5.747%); 8454/231627 objects misplaced (3.650%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T06:41:46.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:46 smithi125 bash[80002]: cluster 2023-10-09T06:41:45.296569+0000 mgr.y (mgr.25723) 1114 : cluster 0 pgmap v1178: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 13312/231627 objects degraded (5.747%); 8454/231627 objects misplaced (3.650%); 6.8 MiB/s, 111 objects/s recovering 2023-10-09T06:41:46.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:46.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:46.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:46.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:46.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:47.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:47.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.180 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:47.284 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:47.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:47.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:47.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:47.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:47.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:47.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:47.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:47.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:47.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:47.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:47.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:48.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:48.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:48.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:48.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:48.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:48.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:48.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:48.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[77299]: audit 2023-10-09T06:41:47.126038+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[77299]: audit 2023-10-09T06:41:47.131059+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[77299]: audit 2023-10-09T06:41:47.255323+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[77299]: audit 2023-10-09T06:41:47.260958+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[77299]: cluster 2023-10-09T06:41:47.297242+0000 mgr.y (mgr.25723) 1115 : cluster 0 pgmap v1179: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 13312/230529 objects degraded (5.775%); 8454/230529 objects misplaced (3.667%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:41:48.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[80002]: audit 2023-10-09T06:41:47.126038+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[80002]: audit 2023-10-09T06:41:47.131059+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[80002]: audit 2023-10-09T06:41:47.255323+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[80002]: audit 2023-10-09T06:41:47.260958+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.373 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:48 smithi125 bash[80002]: cluster 2023-10-09T06:41:47.297242+0000 mgr.y (mgr.25723) 1115 : cluster 0 pgmap v1179: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 13312/230529 objects degraded (5.775%); 8454/230529 objects misplaced (3.667%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:41:48.379 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:48 smithi155 bash[59495]: audit 2023-10-09T06:41:47.126038+0000 mon.a (mon.0) 1362 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.379 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:48 smithi155 bash[59495]: audit 2023-10-09T06:41:47.131059+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.379 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:48 smithi155 bash[59495]: audit 2023-10-09T06:41:47.255323+0000 mon.a (mon.0) 1364 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.379 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:48 smithi155 bash[59495]: audit 2023-10-09T06:41:47.260958+0000 mon.a (mon.0) 1365 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:48.380 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:48 smithi155 bash[59495]: cluster 2023-10-09T06:41:47.297242+0000 mgr.y (mgr.25723) 1115 : cluster 0 pgmap v1179: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 13312/230529 objects degraded (5.775%); 8454/230529 objects misplaced (3.667%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T06:41:48.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:48.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:48.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:48.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:48.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:48.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:48.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:48.824 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:48.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:48.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:48.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:49.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:49.069 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:49.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:49.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:49.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:49.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:49.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:49.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:49.452 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[77299]: audit 2023-10-09T06:41:48.275402+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.452 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[77299]: audit 2023-10-09T06:41:48.284079+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.452 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[77299]: audit 2023-10-09T06:41:48.607262+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.453 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[77299]: audit 2023-10-09T06:41:48.608956+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:49.453 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[77299]: cluster 2023-10-09T06:41:49.027695+0000 mon.a (mon.0) 1370 : cluster 3 Health check update: Degraded data redundancy: 13312/230529 objects degraded (5.775%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:49.453 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[80002]: audit 2023-10-09T06:41:48.275402+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.453 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[80002]: audit 2023-10-09T06:41:48.284079+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[80002]: audit 2023-10-09T06:41:48.607262+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[80002]: audit 2023-10-09T06:41:48.608956+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:49.454 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:49 smithi125 bash[80002]: cluster 2023-10-09T06:41:49.027695+0000 mon.a (mon.0) 1370 : cluster 3 Health check update: Degraded data redundancy: 13312/230529 objects degraded (5.775%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:49.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:49.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:49.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:49 smithi155 bash[59495]: audit 2023-10-09T06:41:48.275402+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:49 smithi155 bash[59495]: audit 2023-10-09T06:41:48.284079+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:49 smithi155 bash[59495]: audit 2023-10-09T06:41:48.607262+0000 mon.a (mon.0) 1368 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:41:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:49 smithi155 bash[59495]: audit 2023-10-09T06:41:48.608956+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:49 smithi155 bash[59495]: cluster 2023-10-09T06:41:49.027695+0000 mon.a (mon.0) 1370 : cluster 3 Health check update: Degraded data redundancy: 13312/230529 objects degraded (5.775%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:49.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:49.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:49.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:50.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:50.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:50.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:50.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:50.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:50.290 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:50.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:50.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:50.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:50.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:50 smithi125 bash[77299]: cluster 2023-10-09T06:41:49.297845+0000 mgr.y (mgr.25723) 1116 : cluster 0 pgmap v1180: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 13312/227784 objects degraded (5.844%); 7821/227784 objects misplaced (3.434%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:41:50.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:50 smithi125 bash[80002]: cluster 2023-10-09T06:41:49.297845+0000 mgr.y (mgr.25723) 1116 : cluster 0 pgmap v1180: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 13312/227784 objects degraded (5.844%); 7821/227784 objects misplaced (3.434%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:41:50.571 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:50.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:50 smithi155 bash[59495]: cluster 2023-10-09T06:41:49.297845+0000 mgr.y (mgr.25723) 1116 : cluster 0 pgmap v1180: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 13312/227784 objects degraded (5.844%); 7821/227784 objects misplaced (3.434%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:41:50.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:50.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:50.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:50.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:50.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:50.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:50.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:51.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:51.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:51.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:51.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:51.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:51.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:51.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:51.495 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:41:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:41:51] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-10-09T06:41:51.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:51.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:51.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:51.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:51.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:51.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:51.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:51.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:52.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:52.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:52.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:52.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:52.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:52.338 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:52.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:52.363 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:52 smithi155 bash[59495]: cluster 2023-10-09T06:41:51.299142+0000 mgr.y (mgr.25723) 1117 : cluster 0 pgmap v1181: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 13388/223143 objects degraded (6.000%); 7662/223143 objects misplaced (3.434%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:41:52.442 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:52.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:52.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:52.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:52.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:52 smithi125 bash[80002]: cluster 2023-10-09T06:41:51.299142+0000 mgr.y (mgr.25723) 1117 : cluster 0 pgmap v1181: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 13388/223143 objects degraded (6.000%); 7662/223143 objects misplaced (3.434%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:41:52.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:52 smithi125 bash[77299]: cluster 2023-10-09T06:41:51.299142+0000 mgr.y (mgr.25723) 1117 : cluster 0 pgmap v1181: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 13388/223143 objects degraded (6.000%); 7662/223143 objects misplaced (3.434%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:41:52.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:52.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:52.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:52.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:52.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:52.988 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:53.016 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:53.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:53.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:53.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:53.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:53.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:53.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:53.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:53.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:53.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:53.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:53.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:53.630 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:41:53.631 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:41:53.631 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:41:53.631 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:41:53.631 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:41:53.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:53.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:53.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:53.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:53.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:53.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:53.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:54.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:54.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:54.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:54.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:54.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:54.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:54.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:54 smithi125 bash[80002]: cluster 2023-10-09T06:41:53.299675+0000 mgr.y (mgr.25723) 1118 : cluster 0 pgmap v1182: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 13388/219258 objects degraded (6.106%); 6974/219258 objects misplaced (3.181%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:41:54.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:54 smithi125 bash[80002]: cluster 2023-10-09T06:41:54.028697+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13388/219258 objects degraded (6.106%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:54.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:54 smithi125 bash[77299]: cluster 2023-10-09T06:41:53.299675+0000 mgr.y (mgr.25723) 1118 : cluster 0 pgmap v1182: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 13388/219258 objects degraded (6.106%); 6974/219258 objects misplaced (3.181%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:41:54.390 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:54 smithi125 bash[77299]: cluster 2023-10-09T06:41:54.028697+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13388/219258 objects degraded (6.106%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:54.469 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:54.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:54.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:54.651 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:54 smithi155 bash[59495]: cluster 2023-10-09T06:41:53.299675+0000 mgr.y (mgr.25723) 1118 : cluster 0 pgmap v1182: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 582 op/s; 13388/219258 objects degraded (6.106%); 6974/219258 objects misplaced (3.181%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:41:54.651 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:54 smithi155 bash[59495]: cluster 2023-10-09T06:41:54.028697+0000 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13388/219258 objects degraded (6.106%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:54.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:54.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:54.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:54.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:55.025 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:55.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:55.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:55.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:55.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:55.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:55.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:55.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:55.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:55.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:55.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.756 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:55.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:55.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:41:55.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:55.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:55.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:55.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:55.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:56.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:56.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:56.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:56.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:56.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:56.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.391 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:56 smithi125 bash[77299]: cluster 2023-10-09T06:41:55.300413+0000 mgr.y (mgr.25723) 1119 : cluster 0 pgmap v1183: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 618 op/s; 13495/214170 objects degraded (6.301%); 6788/214170 objects misplaced (3.169%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T06:41:56.392 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:56 smithi125 bash[80002]: cluster 2023-10-09T06:41:55.300413+0000 mgr.y (mgr.25723) 1119 : cluster 0 pgmap v1183: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 618 op/s; 13495/214170 objects degraded (6.301%); 6788/214170 objects misplaced (3.169%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T06:41:56.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:56.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:56.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:56 smithi155 bash[59495]: cluster 2023-10-09T06:41:55.300413+0000 mgr.y (mgr.25723) 1119 : cluster 0 pgmap v1183: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 618 op/s; 13495/214170 objects degraded (6.301%); 6788/214170 objects misplaced (3.169%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T06:41:56.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:56.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:56.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:56.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:56.967 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:56.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:57.075 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:57.110 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:57.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:57.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:57.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:57.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:57.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:57.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:57.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:57.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:57.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:57.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:57.726 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:41:57.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:57.795 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:57.815 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:41:57.857 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:41:57.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:57.907 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:41:57.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:57.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:57.951 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:41:57.993 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:41:58.023 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:58.026 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:41:58.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:58.091 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:41:58.093 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.094 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:41:58.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:58.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:58.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:58.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:58.401 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 0.167043 (2ac3) 2023-10-09T06:41:58.409 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:41:58.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:58.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:58.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:58 smithi155 bash[59495]: cluster 2023-10-09T06:41:57.300998+0000 mgr.y (mgr.25723) 1120 : cluster 0 pgmap v1184: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 13495/213108 objects degraded (6.332%); 6788/213108 objects misplaced (3.185%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:41:58.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:58 smithi155 bash[59495]: audit 2023-10-09T06:41:57.664080+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.125:0/240189909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:58.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:58 smithi155 bash[59495]: audit 2023-10-09T06:41:58.086955+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]: dispatch 2023-10-09T06:41:58.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:58 smithi155 bash[59495]: audit 2023-10-09T06:41:58.271751+0000 mon.a (mon.0) 1373 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:58 smithi155 bash[59495]: audit 2023-10-09T06:41:58.273839+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.125:0/3739316041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.599 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:58.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[77299]: cluster 2023-10-09T06:41:57.300998+0000 mgr.y (mgr.25723) 1120 : cluster 0 pgmap v1184: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 13495/213108 objects degraded (6.332%); 6788/213108 objects misplaced (3.185%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:41:58.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[77299]: audit 2023-10-09T06:41:57.664080+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.125:0/240189909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[77299]: audit 2023-10-09T06:41:58.086955+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[77299]: audit 2023-10-09T06:41:58.271751+0000 mon.a (mon.0) 1373 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[77299]: audit 2023-10-09T06:41:58.273839+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.125:0/3739316041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[80002]: cluster 2023-10-09T06:41:57.300998+0000 mgr.y (mgr.25723) 1120 : cluster 0 pgmap v1184: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 514 op/s; 13495/213108 objects degraded (6.332%); 6788/213108 objects misplaced (3.185%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[80002]: audit 2023-10-09T06:41:57.664080+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.125:0/240189909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[80002]: audit 2023-10-09T06:41:58.086955+0000 mon.a (mon.0) 1372 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]: dispatch 2023-10-09T06:41:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[80002]: audit 2023-10-09T06:41:58.271751+0000 mon.a (mon.0) 1373 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:58 smithi125 bash[80002]: audit 2023-10-09T06:41:58.273839+0000 mon.b (mon.1) 32 : audit 1 from='client.? 172.21.15.125:0/3739316041' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:58.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:58.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.858 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:58.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:58.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:58.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:58.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:59.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:59.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:59.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:59.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:59.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:59.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:59.371 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:59.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:59.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:59.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:59.633 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:59.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:58.374931+0000 mon.a (mon.0) 1374 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:59.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: audit 2023-10-09T06:41:58.387483+0000 mon.a (mon.0) 1375 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]': finished 2023-10-09T06:41:59.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: audit 2023-10-09T06:41:58.387565+0000 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:59.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:58.387674+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e257: 8 total, 8 up, 6 in 2023-10-09T06:41:59.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:58.390302+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:58.394402+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.029757+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 13495/213108 objects degraded (6.332%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:41:59 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.035743+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e258: 8 total, 8 up, 6 in 2023-10-09T06:41:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:59.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:58.374931+0000 mon.a (mon.0) 1374 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:59.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: audit 2023-10-09T06:41:58.387483+0000 mon.a (mon.0) 1375 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]': finished 2023-10-09T06:41:59.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: audit 2023-10-09T06:41:58.387565+0000 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:58.387674+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e257: 8 total, 8 up, 6 in 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:58.390302+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:58.394402+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.029757+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 13495/213108 objects degraded (6.332%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.035743+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e258: 8 total, 8 up, 6 in 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:58.374931+0000 mon.a (mon.0) 1374 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: audit 2023-10-09T06:41:58.387483+0000 mon.a (mon.0) 1375 : audit 1 from='client.? 172.21.15.125:0/4052440642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.1670432103403715}]': finished 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: audit 2023-10-09T06:41:58.387565+0000 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:59.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:58.387674+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e257: 8 total, 8 up, 6 in 2023-10-09T06:41:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:58.390302+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:58.394402+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e257 with expected crc 2023-10-09T06:41:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.029757+0000 mon.a (mon.0) 1378 : cluster 3 Health check update: Degraded data redundancy: 13495/213108 objects degraded (6.332%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:41:59.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:41:59 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.035743+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e258: 8 total, 8 up, 6 in 2023-10-09T06:41:59.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:59.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:59.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:59.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:41:59.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:59.979 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.039255+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.039519+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.385 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.041397+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.041965+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.042775+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.043237+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.044269+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.045198+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.046622+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.048534+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.098275+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.386 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:41:59.301386+0000 mgr.y (mgr.25723) 1121 : cluster 0 pgmap v1187: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 13495/210618 objects degraded (6.407%); 6221/210618 objects misplaced (2.954%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:42:00.387 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:00 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.037701+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e259: 8 total, 8 up, 6 in 2023-10-09T06:42:00.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.039255+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.589 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.039519+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.041397+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.041965+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.042775+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.043237+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.044269+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.045198+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.046622+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.048534+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.098275+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:41:59.301386+0000 mgr.y (mgr.25723) 1121 : cluster 0 pgmap v1187: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 13495/210618 objects degraded (6.407%); 6221/210618 objects misplaced (2.954%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.037701+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e259: 8 total, 8 up, 6 in 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.039255+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.039519+0000 osd.4 (osd.4) 201 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.041397+0000 osd.4 (osd.4) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.041965+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.042775+0000 osd.4 (osd.4) 203 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.043237+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.044269+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.045198+0000 osd.4 (osd.4) 204 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.046622+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.048534+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.098275+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e258 with expected crc 2023-10-09T06:42:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:41:59.301386+0000 mgr.y (mgr.25723) 1121 : cluster 0 pgmap v1187: 89 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 13495/210618 objects degraded (6.407%); 6221/210618 objects misplaced (2.954%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:42:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:00 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.037701+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e259: 8 total, 8 up, 6 in 2023-10-09T06:42:00.602 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.704 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:00.736 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:00.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:00.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:00.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:00.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:00.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:00.976 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:01.116 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:01.251 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:01.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:01] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.33.4" 2023-10-09T06:42:01.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:01.443 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:01.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:01.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:01.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:01.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.040736+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.041725+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.044137+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.044306+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.045545+0000 osd.3 (osd.3) 15 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 256'8830 2023-10-09T06:42:01.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.047956+0000 osd.7 (osd.7) 185 : cluster 0 9.3 starting backfill to osd.0 from (0'0,0'0] MAX to 256'8820 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.049564+0000 osd.7 (osd.7) 186 : cluster 0 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 256'8783 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.050126+0000 osd.7 (osd.7) 187 : cluster 0 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 256'8764 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.050308+0000 osd.7 (osd.7) 188 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 256'8683 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.093584+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: cluster 2023-10-09T06:42:00.093645+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: audit 2023-10-09T06:42:01.360813+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: audit 2023-10-09T06:42:01.369463+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: audit 2023-10-09T06:42:01.374751+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:01.684 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:01 smithi155 bash[59495]: audit 2023-10-09T06:42:01.376225+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:01.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:01.792 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:01.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:01.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.040736+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.041725+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.044137+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.044306+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.045545+0000 osd.3 (osd.3) 15 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 256'8830 2023-10-09T06:42:01.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.047956+0000 osd.7 (osd.7) 185 : cluster 0 9.3 starting backfill to osd.0 from (0'0,0'0] MAX to 256'8820 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.049564+0000 osd.7 (osd.7) 186 : cluster 0 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 256'8783 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.050126+0000 osd.7 (osd.7) 187 : cluster 0 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 256'8764 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.050308+0000 osd.7 (osd.7) 188 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 256'8683 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.093584+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: cluster 2023-10-09T06:42:00.093645+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: audit 2023-10-09T06:42:01.360813+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: audit 2023-10-09T06:42:01.369463+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: audit 2023-10-09T06:42:01.374751+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:01.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[77299]: audit 2023-10-09T06:42:01.376225+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.040736+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.041725+0000 osd.4 (osd.4) 205 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.044137+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.044306+0000 osd.4 (osd.4) 206 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.045545+0000 osd.3 (osd.3) 15 : cluster 0 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 256'8830 2023-10-09T06:42:01.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.047956+0000 osd.7 (osd.7) 185 : cluster 0 9.3 starting backfill to osd.0 from (0'0,0'0] MAX to 256'8820 2023-10-09T06:42:01.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.049564+0000 osd.7 (osd.7) 186 : cluster 0 9.d starting backfill to osd.0 from (0'0,0'0] MAX to 256'8783 2023-10-09T06:42:01.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.050126+0000 osd.7 (osd.7) 187 : cluster 0 9.5 starting backfill to osd.3 from (0'0,0'0] MAX to 256'8764 2023-10-09T06:42:01.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.050308+0000 osd.7 (osd.7) 188 : cluster 0 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 256'8683 2023-10-09T06:42:01.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.093584+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: cluster 2023-10-09T06:42:00.093645+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e259 with expected crc 2023-10-09T06:42:01.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: audit 2023-10-09T06:42:01.360813+0000 mon.a (mon.0) 1381 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: audit 2023-10-09T06:42:01.369463+0000 mon.a (mon.0) 1382 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:01.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: audit 2023-10-09T06:42:01.374751+0000 mon.a (mon.0) 1383 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:01.895 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:01 smithi125 bash[80002]: audit 2023-10-09T06:42:01.376225+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:01.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:01.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:01.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:02.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:02.091 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:02.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:02.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:02.328 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:02.352 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:02.457 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:02.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.301884+0000 mgr.y (mgr.25723) 1122 : cluster 0 pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 76 active+clean, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 19081/207003 objects degraded (9.218%); 1743/207003 objects misplaced (0.842%); 6.2 MiB/s, 101 objects/s recovering 2023-10-09T06:42:02.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.395677+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.399797+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.402768+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T06:42:02.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.404034+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: audit 2023-10-09T06:42:01.404352+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:02.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:02 smithi155 bash[59495]: cluster 2023-10-09T06:42:01.406547+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:02.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:02.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:02.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:02.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.301884+0000 mgr.y (mgr.25723) 1122 : cluster 0 pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 76 active+clean, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 19081/207003 objects degraded (9.218%); 1743/207003 objects misplaced (0.842%); 6.2 MiB/s, 101 objects/s recovering 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.395677+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.399797+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.402768+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.404034+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: audit 2023-10-09T06:42:01.404352+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[77299]: cluster 2023-10-09T06:42:01.406547+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.301884+0000 mgr.y (mgr.25723) 1122 : cluster 0 pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 76 active+clean, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 19081/207003 objects degraded (9.218%); 1743/207003 objects misplaced (0.842%); 6.2 MiB/s, 101 objects/s recovering 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.395677+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.399797+0000 osd.4 (osd.4) 207 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.402768+0000 mon.a (mon.0) 1385 : cluster 0 osdmap e260: 8 total, 8 up, 6 in 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.404034+0000 osd.4 (osd.4) 208 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: audit 2023-10-09T06:42:01.404352+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:02 smithi125 bash[80002]: cluster 2023-10-09T06:42:01.406547+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e260 with expected crc 2023-10-09T06:42:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:02.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:02.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:02.898 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:02.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:02.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:03.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:03.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:03.130 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:03.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:03.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:03.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:03.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:03.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:03.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:03.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:03.564 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:03.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:03.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:03 smithi155 bash[59495]: cluster 2023-10-09T06:42:02.408191+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:03 smithi155 bash[59495]: cluster 2023-10-09T06:42:02.408476+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:03 smithi155 bash[59495]: cluster 2023-10-09T06:42:02.408477+0000 mon.a (mon.0) 1387 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T06:42:03.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:03 smithi155 bash[59495]: cluster 2023-10-09T06:42:02.412735+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:03.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:03.825 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:03.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[77299]: cluster 2023-10-09T06:42:02.408191+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[77299]: cluster 2023-10-09T06:42:02.408476+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[77299]: cluster 2023-10-09T06:42:02.408477+0000 mon.a (mon.0) 1387 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[77299]: cluster 2023-10-09T06:42:02.412735+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[80002]: cluster 2023-10-09T06:42:02.408191+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[80002]: cluster 2023-10-09T06:42:02.408476+0000 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[80002]: cluster 2023-10-09T06:42:02.408477+0000 mon.a (mon.0) 1387 : cluster 0 osdmap e261: 8 total, 8 up, 6 in 2023-10-09T06:42:03.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:03 smithi125 bash[80002]: cluster 2023-10-09T06:42:02.412735+0000 osd.4 (osd.4) 209 : cluster 3 failed to encode map e261 with expected crc 2023-10-09T06:42:03.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:03.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:03.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:04.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:04.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:42:04.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:04.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:04.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:04.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:04.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:04.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.521 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[77299]: cluster 2023-10-09T06:42:03.302546+0000 mgr.y (mgr.25723) 1123 : cluster 0 pgmap v1192: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 32006/205149 objects degraded (15.601%); 5253/205149 objects misplaced (2.561%); 14 MiB/s, 226 objects/s recovering 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[77299]: audit 2023-10-09T06:42:03.605998+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[77299]: audit 2023-10-09T06:42:03.607467+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[77299]: cluster 2023-10-09T06:42:04.030950+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 32006/205149 objects degraded (15.601%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[80002]: cluster 2023-10-09T06:42:03.302546+0000 mgr.y (mgr.25723) 1123 : cluster 0 pgmap v1192: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 32006/205149 objects degraded (15.601%); 5253/205149 objects misplaced (2.561%); 14 MiB/s, 226 objects/s recovering 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[80002]: audit 2023-10-09T06:42:03.605998+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[80002]: audit 2023-10-09T06:42:03.607467+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:04.522 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:04 smithi125 bash[80002]: cluster 2023-10-09T06:42:04.030950+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 32006/205149 objects degraded (15.601%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:04.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:04.645 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.668 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:04 smithi155 bash[59495]: cluster 2023-10-09T06:42:03.302546+0000 mgr.y (mgr.25723) 1123 : cluster 0 pgmap v1192: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 75 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 32006/205149 objects degraded (15.601%); 5253/205149 objects misplaced (2.561%); 14 MiB/s, 226 objects/s recovering 2023-10-09T06:42:04.668 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:04 smithi155 bash[59495]: audit 2023-10-09T06:42:03.605998+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:04.668 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:04 smithi155 bash[59495]: audit 2023-10-09T06:42:03.607467+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:04.668 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:04 smithi155 bash[59495]: cluster 2023-10-09T06:42:04.030950+0000 mon.a (mon.0) 1390 : cluster 3 Health check update: Degraded data redundancy: 32006/205149 objects degraded (15.601%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:04.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:04.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:04.971 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:04.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:05.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:05.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:05.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:05.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:05.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:05.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:05.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:05.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:05.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:05.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:05.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:05.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:05.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:05.872 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:05.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:06.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:06.035 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:06.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:06.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:06.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:06.178 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:42:06.178 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:42:06.267 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:06.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:06.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:06.437 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:06.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:42:06.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:06.551 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:06 smithi155 bash[59495]: cluster 2023-10-09T06:42:05.305107+0000 mgr.y (mgr.25723) 1124 : cluster 0 pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26498/201687 objects degraded (13.138%); 25918/201687 objects misplaced (12.851%); 11 MiB/s, 184 objects/s recovering 2023-10-09T06:42:06.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:06.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:06.684 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:06 smithi125 bash[77299]: cluster 2023-10-09T06:42:05.305107+0000 mgr.y (mgr.25723) 1124 : cluster 0 pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26498/201687 objects degraded (13.138%); 25918/201687 objects misplaced (12.851%); 11 MiB/s, 184 objects/s recovering 2023-10-09T06:42:06.685 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:06 smithi125 bash[80002]: cluster 2023-10-09T06:42:05.305107+0000 mgr.y (mgr.25723) 1124 : cluster 0 pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26498/201687 objects degraded (13.138%); 25918/201687 objects misplaced (12.851%); 11 MiB/s, 184 objects/s recovering 2023-10-09T06:42:06.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:06.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:06.857 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:06.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:06.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:06.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:07.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:07.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:07.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:07.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:07.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:07.397 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:07.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:42:07.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:07.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:07.638 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:07.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:07.769 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:07.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:07.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:07.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:07.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:08.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:08.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:08.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:08.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:08.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:08.323 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:08.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:08.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:08.579 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:08 smithi155 bash[59495]: cluster 2023-10-09T06:42:07.305670+0000 mgr.y (mgr.25723) 1125 : cluster 0 pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 25872/200742 objects degraded (12.888%); 25918/200742 objects misplaced (12.911%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:42:08.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.736 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:08 smithi125 bash[77299]: cluster 2023-10-09T06:42:07.305670+0000 mgr.y (mgr.25723) 1125 : cluster 0 pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 25872/200742 objects degraded (12.888%); 25918/200742 objects misplaced (12.911%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:42:08.736 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:08 smithi125 bash[80002]: cluster 2023-10-09T06:42:07.305670+0000 mgr.y (mgr.25723) 1125 : cluster 0 pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 25872/200742 objects degraded (12.888%); 25918/200742 objects misplaced (12.911%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:42:08.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:08.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:08.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:08.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:08.977 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:08.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:09.030 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:09.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:09.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:09.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:09.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:09.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:09.365 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:42:09.365 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:42:09.365 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T06:42:09.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T06:42:09.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:09.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:09.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:09.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:09 smithi125 bash[77299]: cluster 2023-10-09T06:42:09.032005+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 25872/200742 objects degraded (12.888%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:09.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:09 smithi125 bash[80002]: cluster 2023-10-09T06:42:09.032005+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 25872/200742 objects degraded (12.888%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:09 smithi155 bash[59495]: cluster 2023-10-09T06:42:09.032005+0000 mon.a (mon.0) 1391 : cluster 3 Health check update: Degraded data redundancy: 25872/200742 objects degraded (12.888%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:09.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:09.699 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:09.802 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:09.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:09.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:09.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:09.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:10.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:10.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:10.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:10.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:10.259 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:10.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:10.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:10.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:10.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:10.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:10.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:10.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:10 smithi125 bash[77299]: cluster 2023-10-09T06:42:09.306397+0000 mgr.y (mgr.25723) 1126 : cluster 0 pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 363 op/s; 25872/198282 objects degraded (13.048%); 25717/198282 objects misplaced (12.970%); 10 MiB/s, 162 objects/s recovering 2023-10-09T06:42:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:10 smithi125 bash[80002]: cluster 2023-10-09T06:42:09.306397+0000 mgr.y (mgr.25723) 1126 : cluster 0 pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 363 op/s; 25872/198282 objects degraded (13.048%); 25717/198282 objects misplaced (12.970%); 10 MiB/s, 162 objects/s recovering 2023-10-09T06:42:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:10 smithi155 bash[59495]: cluster 2023-10-09T06:42:09.306397+0000 mgr.y (mgr.25723) 1126 : cluster 0 pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 363 op/s; 25872/198282 objects degraded (13.048%); 25717/198282 objects misplaced (12.970%); 10 MiB/s, 162 objects/s recovering 2023-10-09T06:42:10.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:10.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:10.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:10.903 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:10.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:10.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:11.010 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:11.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:11.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:11.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:11.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:11.292 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:11.301 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:42:11.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:11.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:11.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:11.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:11.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:11.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:11.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:11.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:11.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:11.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:11.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:12.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:12.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:12.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:12.253 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:12.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:12.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:12.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:12.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:12.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:12.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:12.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:12.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:12.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:12.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:12 smithi125 bash[77299]: cluster 2023-10-09T06:42:11.307027+0000 mgr.y (mgr.25723) 1127 : cluster 0 pgmap v1196: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 25917/193875 objects degraded (13.368%); 24953/193875 objects misplaced (12.871%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T06:42:12.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:12 smithi125 bash[80002]: cluster 2023-10-09T06:42:11.307027+0000 mgr.y (mgr.25723) 1127 : cluster 0 pgmap v1196: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 25917/193875 objects degraded (13.368%); 24953/193875 objects misplaced (12.871%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T06:42:12.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:12.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:12.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:13.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:13.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:13.087 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:12 smithi155 bash[59495]: cluster 2023-10-09T06:42:11.307027+0000 mgr.y (mgr.25723) 1127 : cluster 0 pgmap v1196: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 25917/193875 objects degraded (13.368%); 24953/193875 objects misplaced (12.871%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T06:42:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:13.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:13.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:13.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:13.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:13.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:13.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:13.572 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:13.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:13.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:13.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:13.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:13.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:14.083 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:14.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:14.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:14.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:14.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:14.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:14.490 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:14.530 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:14.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:14.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:14 smithi155 bash[59495]: cluster 2023-10-09T06:42:13.307672+0000 mgr.y (mgr.25723) 1128 : cluster 0 pgmap v1197: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 25327/190530 objects degraded (13.293%); 24746/190530 objects misplaced (12.988%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T06:42:14.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:14 smithi155 bash[59495]: cluster 2023-10-09T06:42:14.033072+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 25327/190530 objects degraded (13.293%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:14.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:14.767 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:14.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:14 smithi125 bash[77299]: cluster 2023-10-09T06:42:13.307672+0000 mgr.y (mgr.25723) 1128 : cluster 0 pgmap v1197: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 25327/190530 objects degraded (13.293%); 24746/190530 objects misplaced (12.988%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T06:42:14.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:14 smithi125 bash[77299]: cluster 2023-10-09T06:42:14.033072+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 25327/190530 objects degraded (13.293%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:14.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:14 smithi125 bash[80002]: cluster 2023-10-09T06:42:13.307672+0000 mgr.y (mgr.25723) 1128 : cluster 0 pgmap v1197: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 25327/190530 objects degraded (13.293%); 24746/190530 objects misplaced (12.988%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T06:42:14.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:14 smithi125 bash[80002]: cluster 2023-10-09T06:42:14.033072+0000 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 25327/190530 objects degraded (13.293%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:14.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:14.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:15.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:15.057 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:15.163 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:15.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:15.287 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:15.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:15.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:15.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:15.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:15.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:15.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:15.698 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:15.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:15.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:15.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:15.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:15.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:15.960 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:15.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:16.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:16.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:16.185 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:16.313 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:16.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.471 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:16.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:16.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.722 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:16 smithi155 bash[59495]: cluster 2023-10-09T06:42:15.308301+0000 mgr.y (mgr.25723) 1129 : cluster 0 pgmap v1198: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 25388/186165 objects degraded (13.637%); 23936/186165 objects misplaced (12.857%); 4.8 MiB/s, 78 objects/s recovering 2023-10-09T06:42:16.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:16.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:16.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:16.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:16.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:16 smithi125 bash[77299]: cluster 2023-10-09T06:42:15.308301+0000 mgr.y (mgr.25723) 1129 : cluster 0 pgmap v1198: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 25388/186165 objects degraded (13.637%); 23936/186165 objects misplaced (12.857%); 4.8 MiB/s, 78 objects/s recovering 2023-10-09T06:42:16.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:16 smithi125 bash[80002]: cluster 2023-10-09T06:42:15.308301+0000 mgr.y (mgr.25723) 1129 : cluster 0 pgmap v1198: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 25388/186165 objects degraded (13.637%); 23936/186165 objects misplaced (12.857%); 4.8 MiB/s, 78 objects/s recovering 2023-10-09T06:42:16.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:17.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:17.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:17.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:17.236 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:17.264 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:42:17.264 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:42:17.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:17.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:17.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:17.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:17.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:17.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:17.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:17.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:17.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:17.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:17.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:18.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:18.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:18.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:18.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:18.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:18.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:18.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:18.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:18.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:42:18.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:18.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:18.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:18 smithi155 bash[59495]: cluster 2023-10-09T06:42:17.308932+0000 mgr.y (mgr.25723) 1130 : cluster 0 pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 24805/185292 objects degraded (13.387%); 23936/185292 objects misplaced (12.918%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:42:18.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:18.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:18.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:18 smithi125 bash[77299]: cluster 2023-10-09T06:42:17.308932+0000 mgr.y (mgr.25723) 1130 : cluster 0 pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 24805/185292 objects degraded (13.387%); 23936/185292 objects misplaced (12.918%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:42:18.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:18 smithi125 bash[80002]: cluster 2023-10-09T06:42:17.308932+0000 mgr.y (mgr.25723) 1130 : cluster 0 pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 24805/185292 objects degraded (13.387%); 23936/185292 objects misplaced (12.918%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:42:18.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:19.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:19.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:19.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:19.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:19.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:19.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:19.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:19.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:19.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:19.561 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:19.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:19.675 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:19 smithi155 bash[59495]: audit 2023-10-09T06:42:18.601900+0000 mon.a (mon.0) 1393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:19.676 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:19 smithi155 bash[59495]: cluster 2023-10-09T06:42:19.033926+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 24805/185292 objects degraded (13.387%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:19.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:19.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:19.823 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:19.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:19 smithi125 bash[77299]: audit 2023-10-09T06:42:18.601900+0000 mon.a (mon.0) 1393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:19.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:19 smithi125 bash[77299]: cluster 2023-10-09T06:42:19.033926+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 24805/185292 objects degraded (13.387%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:19.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:19 smithi125 bash[80002]: audit 2023-10-09T06:42:18.601900+0000 mon.a (mon.0) 1393 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:19.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:19 smithi125 bash[80002]: cluster 2023-10-09T06:42:19.033926+0000 mon.a (mon.0) 1394 : cluster 3 Health check update: Degraded data redundancy: 24805/185292 objects degraded (13.387%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:19.927 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:19.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:19.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:20.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:20.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:20.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:20.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:20.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:20.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:20.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:20.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:20.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:20.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:20.614 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:20 smithi125 bash[77299]: cluster 2023-10-09T06:42:19.309466+0000 mgr.y (mgr.25723) 1131 : cluster 0 pgmap v1200: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 24805/182970 objects degraded (13.557%); 23747/182970 objects misplaced (12.979%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:20.614 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:20 smithi125 bash[80002]: cluster 2023-10-09T06:42:19.309466+0000 mgr.y (mgr.25723) 1131 : cluster 0 pgmap v1200: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 24805/182970 objects degraded (13.557%); 23747/182970 objects misplaced (12.979%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:20.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:20.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:20.737 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:20.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:20.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:20.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:20 smithi155 bash[59495]: cluster 2023-10-09T06:42:19.309466+0000 mgr.y (mgr.25723) 1131 : cluster 0 pgmap v1200: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 24805/182970 objects degraded (13.557%); 23747/182970 objects misplaced (12.979%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:20.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:20.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:21.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:21.029 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:21.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:21.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:21.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:21.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:21.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:21.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:21.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:21.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:21.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:21.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:21.607 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:21] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-10-09T06:42:21.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:21.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:21.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:21.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:21.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:21 smithi125 bash[77299]: cluster 2023-10-09T06:42:21.310061+0000 mgr.y (mgr.25723) 1132 : cluster 0 pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 24824/178833 objects degraded (13.881%); 22984/178833 objects misplaced (12.852%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:42:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:21 smithi125 bash[80002]: cluster 2023-10-09T06:42:21.310061+0000 mgr.y (mgr.25723) 1132 : cluster 0 pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 24824/178833 objects degraded (13.881%); 22984/178833 objects misplaced (12.852%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:42:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:21 smithi155 bash[59495]: cluster 2023-10-09T06:42:21.310061+0000 mgr.y (mgr.25723) 1132 : cluster 0 pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 24824/178833 objects degraded (13.881%); 22984/178833 objects misplaced (12.852%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:42:21.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:21.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:21.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:22.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:22.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:22.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:22.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:22.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:22.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:22.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:22.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:22.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:22.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:22.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:22.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:22.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:22.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:22.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:22.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:22.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:23.018 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:23.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:23.159 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:23.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:23.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:23.314 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:23.319 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:23.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:23.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:23.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:23.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:23.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:23.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:23.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:23.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:23.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:23.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:23.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:23.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:23.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:23.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:24.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:24.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:24.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:24.242 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:24.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:24.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:24.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:24.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:24.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:24.508 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:24 smithi155 bash[59495]: cluster 2023-10-09T06:42:23.310722+0000 mgr.y (mgr.25723) 1133 : cluster 0 pgmap v1202: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24196/175638 objects degraded (13.776%); 22806/175638 objects misplaced (12.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:42:24.508 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:24 smithi155 bash[59495]: cluster 2023-10-09T06:42:24.035101+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 24196/175638 objects degraded (13.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:24.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:24.634 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:24 smithi125 bash[77299]: cluster 2023-10-09T06:42:23.310722+0000 mgr.y (mgr.25723) 1133 : cluster 0 pgmap v1202: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24196/175638 objects degraded (13.776%); 22806/175638 objects misplaced (12.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:42:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:24 smithi125 bash[77299]: cluster 2023-10-09T06:42:24.035101+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 24196/175638 objects degraded (13.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:24.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:24 smithi125 bash[80002]: cluster 2023-10-09T06:42:23.310722+0000 mgr.y (mgr.25723) 1133 : cluster 0 pgmap v1202: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 24196/175638 objects degraded (13.776%); 22806/175638 objects misplaced (12.985%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:42:24.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:24 smithi125 bash[80002]: cluster 2023-10-09T06:42:24.035101+0000 mon.a (mon.0) 1395 : cluster 3 Health check update: Degraded data redundancy: 24196/175638 objects degraded (13.776%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:24.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:24.758 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:24.861 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:24.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:24.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:25.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:25.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:25.134 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:25.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:25.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:25.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:25.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:25.537 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:25.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:25.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:25.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:25.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:25.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:25.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:25.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:26.059 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:26.082 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:26.185 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:26.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:26.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:26.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:26.403 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:42:26.450 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:26.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:26.478 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:42:26.532 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:42:26.575 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:42:26.578 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:26.606 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:42:26.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:26 smithi155 bash[59495]: cluster 2023-10-09T06:42:25.311428+0000 mgr.y (mgr.25723) 1134 : cluster 0 pgmap v1203: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24228/171297 objects degraded (14.144%); 22031/171297 objects misplaced (12.861%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:42:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:26 smithi155 bash[59495]: audit 2023-10-09T06:42:25.909179+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.125:0/1542998545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:26 smithi155 bash[59495]: audit 2023-10-09T06:42:25.909801+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.655 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:42:26.698 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:42:26.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.738 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:42:26.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:26.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:42:26.786 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.786 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.786 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.786 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.787 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.787 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.787 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.787 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:42:26.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[77299]: cluster 2023-10-09T06:42:25.311428+0000 mgr.y (mgr.25723) 1134 : cluster 0 pgmap v1203: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24228/171297 objects degraded (14.144%); 22031/171297 objects misplaced (12.861%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:42:26.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[77299]: audit 2023-10-09T06:42:25.909179+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.125:0/1542998545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[77299]: audit 2023-10-09T06:42:25.909801+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[80002]: cluster 2023-10-09T06:42:25.311428+0000 mgr.y (mgr.25723) 1134 : cluster 0 pgmap v1203: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 24228/171297 objects degraded (14.144%); 22031/171297 objects misplaced (12.861%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:42:26.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[80002]: audit 2023-10-09T06:42:25.909179+0000 mon.c (mon.2) 106 : audit 1 from='client.? 172.21.15.125:0/1542998545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:26 smithi125 bash[80002]: audit 2023-10-09T06:42:25.909801+0000 mon.a (mon.0) 1396 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:26.871 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:26.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:26.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:27.002 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:27.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:27.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:27.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:27.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:27.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:27.373 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:27.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:27.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:27.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[77299]: cluster 2023-10-09T06:42:26.371701+0000 mon.a (mon.0) 1397 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:27.514 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:27.544 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:27.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:27.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:27.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[77299]: audit 2023-10-09T06:42:26.379858+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:27.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[77299]: cluster 2023-10-09T06:42:26.380348+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T06:42:27.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[77299]: cluster 2023-10-09T06:42:26.380373+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T06:42:27.706 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[77299]: audit 2023-10-09T06:42:26.404457+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.125:0/1234004210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:27.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[80002]: cluster 2023-10-09T06:42:26.371701+0000 mon.a (mon.0) 1397 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:27.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[80002]: audit 2023-10-09T06:42:26.379858+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:27.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[80002]: cluster 2023-10-09T06:42:26.380348+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T06:42:27.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[80002]: cluster 2023-10-09T06:42:26.380373+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T06:42:27.707 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:27 smithi125 bash[80002]: audit 2023-10-09T06:42:26.404457+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.125:0/1234004210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:27.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:27.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:27 smithi155 bash[59495]: cluster 2023-10-09T06:42:26.371701+0000 mon.a (mon.0) 1397 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:27.786 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:27 smithi155 bash[59495]: audit 2023-10-09T06:42:26.379858+0000 mon.a (mon.0) 1398 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:27.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:27 smithi155 bash[59495]: cluster 2023-10-09T06:42:26.380348+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e262: 8 total, 8 up, 6 in 2023-10-09T06:42:27.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:27 smithi155 bash[59495]: cluster 2023-10-09T06:42:26.380373+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e262 with expected crc 2023-10-09T06:42:27.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:27 smithi155 bash[59495]: audit 2023-10-09T06:42:26.404457+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.125:0/1234004210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:27.803 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:27.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:27.933 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:27.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:28.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:28.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:28.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:28.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:28.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:28.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.461 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:28.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:28.574 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:28 smithi125 bash[77299]: cluster 2023-10-09T06:42:27.314960+0000 mgr.y (mgr.25723) 1135 : cluster 0 pgmap v1205: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 23602/170358 objects degraded (13.854%); 22031/170358 objects misplaced (12.932%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:28.574 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:28 smithi125 bash[80002]: cluster 2023-10-09T06:42:27.314960+0000 mgr.y (mgr.25723) 1135 : cluster 0 pgmap v1205: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 23602/170358 objects degraded (13.854%); 22031/170358 objects misplaced (12.932%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:28.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:28 smithi155 bash[59495]: cluster 2023-10-09T06:42:27.314960+0000 mgr.y (mgr.25723) 1135 : cluster 0 pgmap v1205: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 23602/170358 objects degraded (13.854%); 22031/170358 objects misplaced (12.932%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:28.704 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:28.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:28.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:28.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:28.871 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:42:28.872 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:42:28.872 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:42:28.872 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:42:28.872 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:42:28.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:28.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:28.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:29.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:29.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:29.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:29.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:29.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:29.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:29.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:29.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:29.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:29 smithi125 bash[80002]: cluster 2023-10-09T06:42:29.036111+0000 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 23602/170358 objects degraded (13.854%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:29.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:29 smithi125 bash[77299]: cluster 2023-10-09T06:42:29.036111+0000 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 23602/170358 objects degraded (13.854%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:29.642 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:29.672 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:29.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:29.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:29.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:29 smithi155 bash[59495]: cluster 2023-10-09T06:42:29.036111+0000 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 23602/170358 objects degraded (13.854%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:29.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:29.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:29.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:29.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:42:30.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:30.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:30.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:30.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:30.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:30.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:30.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:30.421 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:30 smithi125 bash[77299]: cluster 2023-10-09T06:42:29.315557+0000 mgr.y (mgr.25723) 1136 : cluster 0 pgmap v1206: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 23602/167847 objects degraded (14.062%); 21802/167847 objects misplaced (12.989%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:30.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:30.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:30.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:30.606 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:30 smithi155 bash[59495]: cluster 2023-10-09T06:42:29.315557+0000 mgr.y (mgr.25723) 1136 : cluster 0 pgmap v1206: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 23602/167847 objects degraded (14.062%); 21802/167847 objects misplaced (12.989%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:30.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:30.724 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:30 smithi125 bash[80002]: cluster 2023-10-09T06:42:29.315557+0000 mgr.y (mgr.25723) 1136 : cluster 0 pgmap v1206: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 23602/167847 objects degraded (14.062%); 21802/167847 objects misplaced (12.989%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:42:30.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:30.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:30.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:30.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:31.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:31.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:31.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:31.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:31.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:31.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.294 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:31] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-10-09T06:42:31.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:31.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:31.437 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.6. 2023-10-09T06:42:31.459 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:31.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:31.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:31.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[77299]: debug 2023-10-09T06:42:31.387+0000 7ff949fd9700 -1 mon.a@0(leader).osd e262 definitely_dead 0 2023-10-09T06:42:31.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[77299]: audit 2023-10-09T06:42:31.393162+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.125:0/4112969207' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[77299]: audit 2023-10-09T06:42:31.394013+0000 mon.a (mon.0) 1401 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[80002]: audit 2023-10-09T06:42:31.393162+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.125:0/4112969207' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:31 smithi125 bash[80002]: audit 2023-10-09T06:42:31.394013+0000 mon.a (mon.0) 1401 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.721 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:31.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:31.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:31.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:31 smithi155 bash[59495]: audit 2023-10-09T06:42:31.393162+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.125:0/4112969207' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:31 smithi155 bash[59495]: audit 2023-10-09T06:42:31.394013+0000 mon.a (mon.0) 1401 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:42:31.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:31.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:31.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:32.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:32.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:32.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:32.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:32.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:32.384 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:32.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:32.516 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:32.621 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: cluster 2023-10-09T06:42:31.316249+0000 mgr.y (mgr.25723) 1137 : cluster 0 pgmap v1207: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 23657/163626 objects degraded (14.458%); 21040/163626 objects misplaced (12.859%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: cluster 2023-10-09T06:42:31.424553+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: audit 2023-10-09T06:42:31.424721+0000 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: cluster 2023-10-09T06:42:31.424885+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: cluster 2023-10-09T06:42:31.432519+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:32 smithi155 bash[59495]: cluster 2023-10-09T06:42:31.435150+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:42:32.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:32.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: cluster 2023-10-09T06:42:31.316249+0000 mgr.y (mgr.25723) 1137 : cluster 0 pgmap v1207: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 23657/163626 objects degraded (14.458%); 21040/163626 objects misplaced (12.859%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: cluster 2023-10-09T06:42:31.424553+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: audit 2023-10-09T06:42:31.424721+0000 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: cluster 2023-10-09T06:42:31.424885+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: cluster 2023-10-09T06:42:31.432519+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[77299]: cluster 2023-10-09T06:42:31.435150+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: cluster 2023-10-09T06:42:31.316249+0000 mgr.y (mgr.25723) 1137 : cluster 0 pgmap v1207: 89 pgs: 1 active+recovery_wait+remapped, 78 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 23657/163626 objects degraded (14.458%); 21040/163626 objects misplaced (12.859%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:42:32.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: cluster 2023-10-09T06:42:31.424553+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: audit 2023-10-09T06:42:31.424721+0000 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:42:32.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: cluster 2023-10-09T06:42:31.424885+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:42:32.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: cluster 2023-10-09T06:42:31.432519+0000 osd.4 (osd.4) 210 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:32 smithi125 bash[80002]: cluster 2023-10-09T06:42:31.435150+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e263 with expected crc 2023-10-09T06:42:32.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:32.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:32.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:32.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:33.032 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:33.157 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:33.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:33.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:33.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:33.414 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:33.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:33.490 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T06:42:33.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.437152+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.439625+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.440374+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.441902+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.443607+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.445227+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.445645+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.447140+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.447726+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.448923+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.449332+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.450677+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.450849+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.451881+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.454690+0000 osd.7 (osd.7) 196 : cluster 0 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 262'9765 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:32.454740+0000 osd.7 (osd.7) 197 : cluster 0 9.3 starting backfill to osd.6 from (0'0,0'0] MAX to 262'9768 2023-10-09T06:42:33.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:33 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.439961+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:42:33.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:33.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:33.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:33.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:33.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:33.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.437152+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.439625+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.440374+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:42:33.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.441902+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.443607+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.445227+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.445645+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.447140+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.447726+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.448923+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.449332+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.450677+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.450849+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.451881+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.454690+0000 osd.7 (osd.7) 196 : cluster 0 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 262'9765 2023-10-09T06:42:33.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:32.454740+0000 osd.7 (osd.7) 197 : cluster 0 9.3 starting backfill to osd.6 from (0'0,0'0] MAX to 262'9768 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.439961+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.437152+0000 osd.4 (osd.4) 211 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.439625+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.440374+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.441902+0000 osd.4 (osd.4) 212 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.443607+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.445227+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.445645+0000 osd.4 (osd.4) 213 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.447140+0000 osd.4 (osd.4) 214 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.447726+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.448923+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.449332+0000 osd.4 (osd.4) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.450677+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.450849+0000 osd.4 (osd.4) 216 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.451881+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e264 with expected crc 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.454690+0000 osd.7 (osd.7) 196 : cluster 0 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 262'9765 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:32.454740+0000 osd.7 (osd.7) 197 : cluster 0 9.3 starting backfill to osd.6 from (0'0,0'0] MAX to 262'9768 2023-10-09T06:42:33.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:33 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.439961+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:42:33.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:33.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:33.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:34.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:34.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:34.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:34.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:34.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:34.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:34.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:34.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:34.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:34.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.316771+0000 mgr.y (mgr.25723) 1138 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 75 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 728/161211 objects degraded (0.452%); 19744/161211 objects misplaced (12.247%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:42:34.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.438299+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.442503+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.445552+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.446020+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:33.448060+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: audit 2023-10-09T06:42:33.605800+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: audit 2023-10-09T06:42:33.606985+0000 mon.a (mon.0) 1407 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:34.037156+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 728/161211 objects degraded (0.452%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[77299]: cluster 2023-10-09T06:42:34.444034+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.316771+0000 mgr.y (mgr.25723) 1138 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 75 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 728/161211 objects degraded (0.452%); 19744/161211 objects misplaced (12.247%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:42:34.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.438299+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.442503+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.445552+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.446020+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:33.448060+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: audit 2023-10-09T06:42:33.605800+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: audit 2023-10-09T06:42:33.606985+0000 mon.a (mon.0) 1407 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:34.037156+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 728/161211 objects degraded (0.452%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:34.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:34 smithi125 bash[80002]: cluster 2023-10-09T06:42:34.444034+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:42:34.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:34.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:34.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:34.776 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.316771+0000 mgr.y (mgr.25723) 1138 : cluster 0 pgmap v1210: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 75 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 728/161211 objects degraded (0.452%); 19744/161211 objects misplaced (12.247%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:42:34.777 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.438299+0000 osd.4 (osd.4) 217 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.777 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.442503+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.777 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.445552+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.777 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.446020+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.777 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:33.448060+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e265 with expected crc 2023-10-09T06:42:34.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: audit 2023-10-09T06:42:33.605800+0000 mon.a (mon.0) 1406 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:42:34.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: audit 2023-10-09T06:42:33.606985+0000 mon.a (mon.0) 1407 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:34.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:34.037156+0000 mon.a (mon.0) 1408 : cluster 3 Health check update: Degraded data redundancy: 728/161211 objects degraded (0.452%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:34.778 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:34 smithi155 bash[59495]: cluster 2023-10-09T06:42:34.444034+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:42:34.788 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:34.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:34.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:34.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:34.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:35.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:35.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:35.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:35.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:35.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:35.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:35.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:35.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:35.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:35.611 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:35 smithi155 bash[59495]: cluster 2023-10-09T06:42:34.441702+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:35 smithi155 bash[59495]: cluster 2023-10-09T06:42:34.444409+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:35 smithi155 bash[59495]: cluster 2023-10-09T06:42:34.450084+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:35.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[77299]: cluster 2023-10-09T06:42:34.441702+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[77299]: cluster 2023-10-09T06:42:34.444409+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[77299]: cluster 2023-10-09T06:42:34.450084+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[80002]: cluster 2023-10-09T06:42:34.441702+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[80002]: cluster 2023-10-09T06:42:34.444409+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:35 smithi125 bash[80002]: cluster 2023-10-09T06:42:34.450084+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e266 with expected crc 2023-10-09T06:42:35.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:35.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:35.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:35.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:35.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:35.981 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:36.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:36.110 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:36.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:36.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:36.266 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:36.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:36.396 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:36.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:36.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:36.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:36 smithi155 bash[59495]: cluster 2023-10-09T06:42:35.317535+0000 mgr.y (mgr.25723) 1139 : cluster 0 pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 1927/158835 objects degraded (1.213%); 15950/158835 objects misplaced (10.042%); 13 MiB/s, 210 objects/s recovering 2023-10-09T06:42:36.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:36.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:36.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:36.800 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:36 smithi125 bash[77299]: cluster 2023-10-09T06:42:35.317535+0000 mgr.y (mgr.25723) 1139 : cluster 0 pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 1927/158835 objects degraded (1.213%); 15950/158835 objects misplaced (10.042%); 13 MiB/s, 210 objects/s recovering 2023-10-09T06:42:36.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:36 smithi125 bash[80002]: cluster 2023-10-09T06:42:35.317535+0000 mgr.y (mgr.25723) 1139 : cluster 0 pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 4 peering, 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 399 op/s; 1927/158835 objects degraded (1.213%); 15950/158835 objects misplaced (10.042%); 13 MiB/s, 210 objects/s recovering 2023-10-09T06:42:36.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:36.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:36.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:36.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:37.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:37.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:37.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:37.230 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:37.334 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:37.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:37.428 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:37.429 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:37.463 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:37.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:37.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:37.603 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:37 smithi125 bash[77299]: cluster 2023-10-09T06:42:37.318116+0000 mgr.y (mgr.25723) 1140 : cluster 0 pgmap v1214: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 peering, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 9512/158049 objects degraded (6.018%); 19188/158049 objects misplaced (12.141%); 9.9 MiB/s, 161 objects/s recovering 2023-10-09T06:42:37.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:37 smithi125 bash[80002]: cluster 2023-10-09T06:42:37.318116+0000 mgr.y (mgr.25723) 1140 : cluster 0 pgmap v1214: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 peering, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 9512/158049 objects degraded (6.018%); 19188/158049 objects misplaced (12.141%); 9.9 MiB/s, 161 objects/s recovering 2023-10-09T06:42:37.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:37.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:37.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:37.823 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:37 smithi155 bash[59495]: cluster 2023-10-09T06:42:37.318116+0000 mgr.y (mgr.25723) 1140 : cluster 0 pgmap v1214: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 peering, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 9512/158049 objects degraded (6.018%); 19188/158049 objects misplaced (12.141%); 9.9 MiB/s, 161 objects/s recovering 2023-10-09T06:42:37.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:37.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:37.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:38.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:38.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:38.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:38.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:38.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:38.350 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:38.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:38.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:38.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:38.491 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:42:38.491 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:42:38.491 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-09T06:42:38.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:42:38.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:38.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:38.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:38.733 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:38.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:38.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:38.965 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:38.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:38.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:39.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:39.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.230 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:39 smithi155 bash[59495]: cluster 2023-10-09T06:42:39.038330+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 9512/158049 objects degraded (6.018%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:39.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:108459 8.4 8:22321375:::benchmark_data_smithi125_176369_object108458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:42:39.338 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:25.824062+0000", 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 433.50964070399999, 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.266836195, 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 108459 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.339 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.824062+0000", 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.824062+0000", 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.824061+0000", 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2521000000000005e-05 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.824144+0000", 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003576011 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.827720+0000", 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8880000000000003e-06 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.827728+0000", 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1171999999999997e-05 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.827780+0000", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000137477 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.827917+0000", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000175348 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.828092+0000", 2023-10-09T06:42:39.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007814573 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.835907+0000", 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0070946489999999997 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.843002+0000", 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.247774779 2023-10-09T06:42:39.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090776+0000", 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7389000000000001e-05 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090844+0000", 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4404999999999998e-05 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.345 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090898+0000", 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.266836195 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:108464 8.4 8:23244b30:::benchmark_data_smithi125_176369_object108463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:42:39.346 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:25.832487+0000", 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 433.50121525399999, 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25855767699999999, 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 108464 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.347 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832487+0000", 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832487+0000", 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:39.348 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832487+0000", 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2669e-05 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832540+0000", 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4659999999999998e-06 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832545+0000", 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2049999999999998e-06 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832548+0000", 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5170999999999997e-05 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832603+0000", 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013464400000000001 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832738+0000", 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000165079 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832903+0000", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010102368 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.843005+0000", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.004607926 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.847613+0000", 2023-10-09T06:42:39.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.243333521 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090947+0000", 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7101999999999997e-05 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090994+0000", 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1110999999999999e-05 2023-10-09T06:42:39.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091045+0000", 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25855767699999999 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:108465 8.4 8:200bdf86:::benchmark_data_smithi125_176369_object108464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:42:39.354 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:25.832566+0000", 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 433.50113661199998, 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25861880799999998, 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 108465 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832566+0000", 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832566+0000", 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832565+0000", 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4869999999999994e-05 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832630+0000", 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2119999999999997e-06 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832637+0000", 2023-10-09T06:42:39.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1680000000000003e-06 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832643+0000", 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032801800000000002 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.835923+0000", 2023-10-09T06:42:39.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103213 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.836026+0000", 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000104838 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.836131+0000", 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068793880000000002 2023-10-09T06:42:39.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.843010+0000", 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046203160000000002 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.847631+0000", 2023-10-09T06:42:39.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.243461609 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091092+0000", 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2500999999999998e-05 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091145+0000", 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0272999999999999e-05 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091185+0000", 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25861880799999998 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:108466 8.4 8:247afedc:::benchmark_data_smithi125_176369_object108465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:25.832649+0000", 2023-10-09T06:42:39.362 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 433.501053297, 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25866880599999997, 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 108466 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.363 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832649+0000", 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832649+0000", 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.364 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832649+0000", 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2189999999999999e-05 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832711+0000", 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0169999999999996e-06 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.365 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832719+0000", 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1479999999999997e-06 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.832724+0000", 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0065778019999999998 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.839302+0000", 2023-10-09T06:42:39.366 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000117484 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.839419+0000", 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014958800000000001 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.839569+0000", 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003598291 2023-10-09T06:42:39.367 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.843167+0000", 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.24727459800000001 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090442+0000", 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00078061700000000005 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.368 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.369 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091222+0000", 2023-10-09T06:42:39.369 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5358999999999997e-05 2023-10-09T06:42:39.369 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.369 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.369 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:42:39.371 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091268+0000", 2023-10-09T06:42:39.371 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0392000000000001e-05 2023-10-09T06:42:39.371 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.371 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.371 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091318+0000", 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25866880599999997 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.372 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:108478 8.4 8:286447dd:::benchmark_data_smithi125_176369_object108477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:35:25.848012+0000", 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 433.48569042100002, 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.249538337, 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.373 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.374 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.374 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 108478 2023-10-09T06:42:39.374 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.374 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.374 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848012+0000", 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848012+0000", 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.375 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.376 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.376 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848011+0000", 2023-10-09T06:42:39.376 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6315999999999993e-05 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848077+0000", 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9409999999999996e-06 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848086+0000", 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5260000000000004e-06 2023-10-09T06:42:39.377 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848091+0000", 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.1943000000000002e-05 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848123+0000", 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8827999999999996e-05 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.378 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848221+0000", 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011108900000000001 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:25.848333+0000", 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.24221546099999999 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.379 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.090548+0000", 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00050889700000000002 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.091057+0000", 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0063839309999999998 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.097441+0000", 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0649999999999997e-05 2023-10-09T06:42:39.380 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.097501+0000", 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9157999999999999e-05 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:35:26.097551+0000", 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.249538337 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.381 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217074 8.11 8.a77f47f1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.081835+0000", 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.25186781899998, 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92998152999999995, 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.382 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217074 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081835+0000", 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081835+0000", 2023-10-09T06:42:39.383 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081833+0000", 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.8180000000000004e-06 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081841+0000", 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8590999999999999e-05 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.384 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081859+0000", 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0429999999999996e-06 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.081868+0000", 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0020447199999999999 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.385 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.083913+0000", 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1069999999999994e-06 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.083922+0000", 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92787777299999996 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011800+0000", 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6432000000000001e-05 2023-10-09T06:42:39.386 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011816+0000", 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92998152999999995 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217101 8.11 8.822571b1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.114466+0000", 2023-10-09T06:42:39.387 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.21923650100001, 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89741884999999999, 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217101 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.388 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114466+0000", 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114466+0000", 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114465+0000", 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3170000000000002e-06 2023-10-09T06:42:39.389 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114473+0000", 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4382000000000001e-05 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114487+0000", 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5059999999999998e-06 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.390 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114495+0000", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1285999999999995e-05 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114566+0000", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0995999999999999e-05 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.114577+0000", 2023-10-09T06:42:39.391 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89729371899999999 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011871+0000", 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4344e-05 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011885+0000", 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89741884999999999 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.392 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217222 8.11 8.cec4fff1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.255504+0000", 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.07819885599997, 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.756416113, 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.393 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217222 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255504+0000", 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.394 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255504+0000", 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255502+0000", 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0619999999999997e-06 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255508+0000", 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0412e-05 2023-10-09T06:42:39.395 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255518+0000", 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.75e-06 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255524+0000", 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7774e-05 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.396 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255572+0000", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4179999999999996e-06 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.255577+0000", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.75633128900000002 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011909+0000", 2023-10-09T06:42:39.397 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1426e-05 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011920+0000", 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.756416113 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.398 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217254 8.11 8.52de4091 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.284406+0000", 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.049297072, 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.728070722, 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217254 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.399 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284406+0000", 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284406+0000", 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.400 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284405+0000", 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9689999999999999e-06 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284411+0000", 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2492e-05 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284423+0000", 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6010000000000002e-06 2023-10-09T06:42:39.401 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.284429+0000", 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003221991 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.287651+0000", 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4869999999999996e-06 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.402 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.287658+0000", 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.724297148 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011955+0000", 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00052104199999999999 2023-10-09T06:42:39.403 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012476+0000", 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.728070722 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.404 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217300 8.11 8.549234d1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.330643+0000", 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.003059275, 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.681345914, 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217300 2023-10-09T06:42:39.405 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330643+0000", 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330643+0000", 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.406 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330642+0000", 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1540000000000001e-06 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330647+0000", 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3489999999999998e-06 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.407 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330657+0000", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5529999999999998e-06 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330663+0000", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6841e-05 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330720+0000", 2023-10-09T06:42:39.408 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3940000000000002e-06 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330730+0000", 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.68125111699999996 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011981+0000", 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6799999999999999e-06 2023-10-09T06:42:39.409 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.011989+0000", 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.681345914 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217306 8.11 8.dc8d1571 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.330909+0000", 2023-10-09T06:42:39.410 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 277.00279365099999, 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.68111383000000003, 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217306 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.411 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330909+0000", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330909+0000", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330908+0000", 2023-10-09T06:42:39.412 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2919999999999998e-06 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330915+0000", 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4445e-05 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330930+0000", 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7589999999999997e-06 2023-10-09T06:42:39.413 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.330937+0000", 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023257 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.331170+0000", 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4902999999999999e-05 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.331185+0000", 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.68082805400000002 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012013+0000", 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.927e-06 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.415 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012023+0000", 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.68111383000000003 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217309 8.11 8.dc738031 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.335108+0000", 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.99859515499998, 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.67694372400000002, 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.416 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217309 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335108+0000", 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.417 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335108+0000", 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335106+0000", 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0210000000000002e-06 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.418 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335113+0000", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4363e-05 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335127+0000", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8260000000000004e-06 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335134+0000", 2023-10-09T06:42:39.419 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00075345900000000005 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335887+0000", 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0502999999999999e-05 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.335898+0000", 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.67614620700000005 2023-10-09T06:42:39.420 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012044+0000", 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1879999999999996e-06 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012051+0000", 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.67694372400000002 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.421 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217341 8.11 8.f47d6031 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.374616+0000", 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.95908683499999, 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63746046199999995, 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.422 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217341 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374616+0000", 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374616+0000", 2023-10-09T06:42:39.423 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374614+0000", 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0463e-05 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374634+0000", 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.113e-05 2023-10-09T06:42:39.424 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374645+0000", 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5969999999999997e-06 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374652+0000", 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3023999999999998e-05 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.425 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374715+0000", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374725+0000", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63734434299999998 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012069+0000", 2023-10-09T06:42:39.426 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.131e-06 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012076+0000", 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63746046199999995 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.427 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217344 8.11 8.ffa93c11 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.374843+0000", 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.95885980600002, 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63725828600000001, 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217344 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.428 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374843+0000", 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374843+0000", 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.429 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374842+0000", 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.668e-06 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374847+0000", 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5100000000000004e-06 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374853+0000", 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9349999999999996e-06 2023-10-09T06:42:39.430 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.374857+0000", 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00028961600000000003 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375147+0000", 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1330000000000004e-06 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.431 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375153+0000", 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63694111200000003 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012094+0000", 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1459999999999999e-06 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.432 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012101+0000", 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63725828600000001 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217345 8.11 8.c0125111 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.375675+0000", 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.95802792900002, 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.636453461, 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.433 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217345 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375675+0000", 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.434 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375675+0000", 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375674+0000", 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8990000000000004e-06 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.435 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375681+0000", 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2393e-05 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375693+0000", 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0740000000000004e-06 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375700+0000", 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.981e-05 2023-10-09T06:42:39.436 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375740+0000", 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0329999999999999e-06 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.375745+0000", 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63637574600000002 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.437 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012121+0000", 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6159999999999996e-06 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012128+0000", 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.636453461 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.438 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217422 8.11 8.b0c3ea71 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.470359+0000", 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.86334353299998, 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.54179315400000005, 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.439 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217422 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470359+0000", 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.440 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470359+0000", 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470357+0000", 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.328e-05 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470370+0000", 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8955999999999999e-05 2023-10-09T06:42:39.441 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470389+0000", 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.185e-05 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470401+0000", 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5609999999999998e-05 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.442 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.443 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470456+0000", 2023-10-09T06:42:39.443 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2651e-05 2023-10-09T06:42:39.443 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.470469+0000", 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.54167671699999997 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012146+0000", 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7120000000000003e-06 2023-10-09T06:42:39.445 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.446 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.446 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012152+0000", 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.54179315400000005 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217495 8.11 8.c6d62bd1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.583580+0000", 2023-10-09T06:42:39.448 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.75012240699999, 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.428597746, 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217495 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.449 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583580+0000", 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583580+0000", 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583579+0000", 2023-10-09T06:42:39.450 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2590000000000003e-06 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583586+0000", 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7659000000000002e-05 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583604+0000", 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.9929999999999995e-06 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.451 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583612+0000", 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7509000000000006e-05 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583689+0000", 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0797e-05 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.452 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.583700+0000", 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.42847115000000002 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012171+0000", 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9319999999999999e-06 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012178+0000", 2023-10-09T06:42:39.453 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.428597746 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217565 8.11 8.427de4f1 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.688049+0000", 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.64565386700002, 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.32415317399999999, 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.454 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217565 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688049+0000", 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.455 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688049+0000", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688047+0000", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5820000000000004e-06 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688055+0000", 2023-10-09T06:42:39.456 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3485e-05 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688068+0000", 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0570000000000004e-06 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688075+0000", 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5815000000000005e-05 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.457 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688151+0000", 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3798000000000001e-05 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.688165+0000", 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.324031334 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.458 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012196+0000", 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8250000000000001e-06 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012202+0000", 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.32415317399999999 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.459 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:217595 8.11 8.f91eb931 (undecoded) ondisk+write+known_if_redirected e215)", 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:38:02.740606+0000", 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 276.59309661700001, 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.27161247599999999, 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 217595 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.460 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740606+0000", 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740606+0000", 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:39.461 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740605+0000", 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1960000000000002e-06 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740612+0000", 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3684e-05 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740626+0000", 2023-10-09T06:42:39.462 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3949999999999998e-06 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740632+0000", 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0594999999999998e-05 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740693+0000", 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4169999999999998e-06 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.463 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:02.740702+0000", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.271511371 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012214+0000", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8810000000000004e-06 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:38:03.012219+0000", 2023-10-09T06:42:39.464 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.27161247599999999 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:39.465 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:42:39.465 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:39.466 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:39 smithi125 bash[77299]: cluster 2023-10-09T06:42:39.038330+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 9512/158049 objects degraded (6.018%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:39.466 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:39 smithi125 bash[80002]: cluster 2023-10-09T06:42:39.038330+0000 mon.a (mon.0) 1410 : cluster 3 Health check update: Degraded data redundancy: 9512/158049 objects degraded (6.018%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:39.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:39.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:39.632 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:39.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:39.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:39.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:39.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:39.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:40.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:40.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:40.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:40.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:40 smithi155 bash[59495]: cluster 2023-10-09T06:42:39.318728+0000 mgr.y (mgr.25723) 1141 : cluster 0 pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 9512/155406 objects degraded (6.121%); 18348/155406 objects misplaced (11.806%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:42:40.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:40.171 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:40 smithi125 bash[80002]: cluster 2023-10-09T06:42:39.318728+0000 mgr.y (mgr.25723) 1141 : cluster 0 pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 9512/155406 objects degraded (6.121%); 18348/155406 objects misplaced (11.806%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:42:40.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:40.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:40.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:42:40.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:40.426 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:40 smithi125 bash[77299]: cluster 2023-10-09T06:42:39.318728+0000 mgr.y (mgr.25723) 1141 : cluster 0 pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 9512/155406 objects degraded (6.121%); 18348/155406 objects misplaced (11.806%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:42:40.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:40.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:40.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:40.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:40.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:40.723 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:40.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:40.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:40.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:40.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:41.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:41.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:41.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:41.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:41] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-10-09T06:42:41.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:41.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:41.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:41.349 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:41.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:41.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:41.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:41.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:41.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:41.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:41.775 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:41.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:41.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:42.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:42.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:42.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:42.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:42.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:42.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:42.310 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:42.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:42.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:42.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:42.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:42 smithi125 bash[77299]: cluster 2023-10-09T06:42:41.319414+0000 mgr.y (mgr.25723) 1142 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 9512/149793 objects degraded (6.350%); 17544/149793 objects misplaced (11.712%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T06:42:42.551 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:42 smithi125 bash[80002]: cluster 2023-10-09T06:42:41.319414+0000 mgr.y (mgr.25723) 1142 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 9512/149793 objects degraded (6.350%); 17544/149793 objects misplaced (11.712%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T06:42:42.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:42 smithi155 bash[59495]: cluster 2023-10-09T06:42:41.319414+0000 mgr.y (mgr.25723) 1142 : cluster 0 pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 475 op/s; 9512/149793 objects degraded (6.350%); 17544/149793 objects misplaced (11.712%); 6.9 MiB/s, 112 objects/s recovering 2023-10-09T06:42:42.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:42.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:42.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:42.845 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:42.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:42.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:42.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:42.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:43.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:43.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:43.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.346 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:43.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:43.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:43.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:43.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:43.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:43.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:43.761 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:43.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:43.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:43.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:44.028 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:44.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:44.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:44.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:44.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:44.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:44 smithi125 bash[77299]: cluster 2023-10-09T06:42:43.320098+0000 mgr.y (mgr.25723) 1143 : cluster 0 pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9512/148563 objects degraded (6.403%); 17355/148563 objects misplaced (11.682%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T06:42:44.383 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:44 smithi125 bash[77299]: cluster 2023-10-09T06:42:44.039478+0000 mon.a (mon.0) 1411 : cluster 3 Health check update: Degraded data redundancy: 9512/148563 objects degraded (6.403%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:44.442 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:44.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.593 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:44 smithi155 bash[59495]: cluster 2023-10-09T06:42:43.320098+0000 mgr.y (mgr.25723) 1143 : cluster 0 pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9512/148563 objects degraded (6.403%); 17355/148563 objects misplaced (11.682%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T06:42:44.593 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:44 smithi155 bash[59495]: cluster 2023-10-09T06:42:44.039478+0000 mon.a (mon.0) 1411 : cluster 3 Health check update: Degraded data redundancy: 9512/148563 objects degraded (6.403%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:44.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:44 smithi125 bash[80002]: cluster 2023-10-09T06:42:43.320098+0000 mgr.y (mgr.25723) 1143 : cluster 0 pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9512/148563 objects degraded (6.403%); 17355/148563 objects misplaced (11.682%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T06:42:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:44 smithi125 bash[80002]: cluster 2023-10-09T06:42:44.039478+0000 mon.a (mon.0) 1411 : cluster 3 Health check update: Degraded data redundancy: 9512/148563 objects degraded (6.403%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:44.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.742 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:44.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:44.877 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:44.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:44.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:45.117 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:45.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:45.200 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:45.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:45.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:45.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:45.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:45.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:45.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:45.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:45.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:45.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:45.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:45.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:46.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:46.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:46.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:46.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:46.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:46.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:46.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:46.443 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:46 smithi125 bash[77299]: cluster 2023-10-09T06:42:45.321106+0000 mgr.y (mgr.25723) 1144 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 9512/140211 objects degraded (6.784%); 15950/140211 objects misplaced (11.376%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T06:42:46.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:46 smithi125 bash[80002]: cluster 2023-10-09T06:42:45.321106+0000 mgr.y (mgr.25723) 1144 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 9512/140211 objects degraded (6.784%); 15950/140211 objects misplaced (11.376%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T06:42:46.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:46.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:46.591 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:46.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:46.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:46.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:46.842 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:46 smithi155 bash[59495]: cluster 2023-10-09T06:42:45.321106+0000 mgr.y (mgr.25723) 1144 : cluster 0 pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 9512/140211 objects degraded (6.784%); 15950/140211 objects misplaced (11.376%); 9.3 MiB/s, 150 objects/s recovering 2023-10-09T06:42:46.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:46.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:46.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:46.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:47.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:47.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:47.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:47.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:47.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:47.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:47.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:47.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:47.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:47.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:47.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:47.777 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:47.931 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:47.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:48.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:48.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:48.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:48.298 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:48.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:48.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:48.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:48.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:48.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.582 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:48 smithi155 bash[59495]: cluster 2023-10-09T06:42:47.321579+0000 mgr.y (mgr.25723) 1145 : cluster 0 pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 9512/139035 objects degraded (6.841%); 15753/139035 objects misplaced (11.330%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:48.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:48.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:48 smithi125 bash[77299]: cluster 2023-10-09T06:42:47.321579+0000 mgr.y (mgr.25723) 1145 : cluster 0 pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 9512/139035 objects degraded (6.841%); 15753/139035 objects misplaced (11.330%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:48.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:48 smithi125 bash[80002]: cluster 2023-10-09T06:42:47.321579+0000 mgr.y (mgr.25723) 1145 : cluster 0 pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 9512/139035 objects degraded (6.841%); 15753/139035 objects misplaced (11.330%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:48.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:48.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:48.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:48.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:48.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:49.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:49.127 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:49.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:49.306 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:49.327 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:49.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:49.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:49.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:49.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:49.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:49 smithi155 bash[59495]: audit 2023-10-09T06:42:48.602642+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:49.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:49 smithi155 bash[59495]: cluster 2023-10-09T06:42:49.040521+0000 mon.a (mon.0) 1413 : cluster 3 Health check update: Degraded data redundancy: 9512/139035 objects degraded (6.841%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:49.717 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:49.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:49.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:49 smithi125 bash[77299]: audit 2023-10-09T06:42:48.602642+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:49.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:49 smithi125 bash[77299]: cluster 2023-10-09T06:42:49.040521+0000 mon.a (mon.0) 1413 : cluster 3 Health check update: Degraded data redundancy: 9512/139035 objects degraded (6.841%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:49.841 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:49 smithi125 bash[80002]: audit 2023-10-09T06:42:48.602642+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:49.842 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:49 smithi125 bash[80002]: cluster 2023-10-09T06:42:49.040521+0000 mon.a (mon.0) 1413 : cluster 3 Health check update: Degraded data redundancy: 9512/139035 objects degraded (6.841%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:49.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:49.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:49.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:49.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:49.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:50.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:50.144 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:50.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:50.274 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:50.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:50.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:50.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:50.536 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:42:50.536 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_repop(client.45605.0:166292 9.6 e266/263 9:632b0776:::benchmark_data_smithi125_199533_object61761:head v 266'10148, mlcod=248'3719)", 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:42:50.531215+0000", 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0023636289999999999, 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023938459999999998, 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:42:50.537 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:42:50.538 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.538 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:42:50.538 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531215+0000", 2023-10-09T06:42:50.538 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:42:50.538 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.540 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531215+0000", 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531214+0000", 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5590000000000002e-06 2023-10-09T06:42:50.541 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531221+0000", 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9750000000000002e-06 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531231+0000", 2023-10-09T06:42:50.542 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4380000000000003e-06 2023-10-09T06:42:50.543 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.543 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531236+0000", 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8965000000000003e-05 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531285+0000", 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6783000000000001e-05 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:42:50.544 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:50.531312+0000", 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6879999999999994e-05 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 1 2023-10-09T06:42:50.545 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:42:50.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:50 smithi155 bash[59495]: cluster 2023-10-09T06:42:49.324487+0000 mgr.y (mgr.25723) 1146 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 9512/136218 objects degraded (6.983%); 15162/136218 objects misplaced (11.131%); 11 MiB/s, 178 objects/s recovering 2023-10-09T06:42:50.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:50.661 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:50 smithi125 bash[77299]: cluster 2023-10-09T06:42:49.324487+0000 mgr.y (mgr.25723) 1146 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 9512/136218 objects degraded (6.983%); 15162/136218 objects misplaced (11.131%); 11 MiB/s, 178 objects/s recovering 2023-10-09T06:42:50.662 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:50 smithi125 bash[80002]: cluster 2023-10-09T06:42:49.324487+0000 mgr.y (mgr.25723) 1146 : cluster 0 pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 9512/136218 objects degraded (6.983%); 15162/136218 objects misplaced (11.131%); 11 MiB/s, 178 objects/s recovering 2023-10-09T06:42:50.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:50.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:50.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:50.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:50.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:50.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:51.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:51.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:51.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:51.346 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:51.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:51.478 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:51.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:51.499 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:42:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:42:51] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.33.4" 2023-10-09T06:42:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:51.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:51.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:51.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:42:51.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:51.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:51.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:52.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:52.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:52.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:52.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:52.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:52.267 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:52.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:52.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:52.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:52.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:52.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:52 smithi125 bash[77299]: cluster 2023-10-09T06:42:51.325203+0000 mgr.y (mgr.25723) 1147 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 9512/131253 objects degraded (7.247%); 14424/131253 objects misplaced (10.989%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T06:42:52.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:52 smithi125 bash[80002]: cluster 2023-10-09T06:42:51.325203+0000 mgr.y (mgr.25723) 1147 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 9512/131253 objects degraded (7.247%); 14424/131253 objects misplaced (10.989%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T06:42:52.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:52.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:52.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:52.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:52.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:52 smithi155 bash[59495]: cluster 2023-10-09T06:42:51.325203+0000 mgr.y (mgr.25723) 1147 : cluster 0 pgmap v1221: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 9512/131253 objects degraded (7.247%); 14424/131253 objects misplaced (10.989%); 7.5 MiB/s, 120 objects/s recovering 2023-10-09T06:42:52.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:52.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:52.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:52.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:53.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:53.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:53.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:53.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:53.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:53.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:53.376 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:53.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:53.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:53.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:53.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:53.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:53.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:53.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:53.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:53.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:53.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:54.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:54.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:54.190 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:54.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:54.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:54.346 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:54.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:54.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:54 smithi155 bash[59495]: cluster 2023-10-09T06:42:53.325818+0000 mgr.y (mgr.25723) 1148 : cluster 0 pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 9512/130122 objects degraded (7.310%); 14238/130122 objects misplaced (10.942%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:42:54.457 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:54 smithi155 bash[59495]: cluster 2023-10-09T06:42:54.044591+0000 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 9512/130122 objects degraded (7.310%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:54.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:54.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:54.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:54.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:54.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:54.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:54 smithi125 bash[80002]: cluster 2023-10-09T06:42:53.325818+0000 mgr.y (mgr.25723) 1148 : cluster 0 pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 9512/130122 objects degraded (7.310%); 14238/130122 objects misplaced (10.942%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:42:54.725 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:54 smithi125 bash[80002]: cluster 2023-10-09T06:42:54.044591+0000 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 9512/130122 objects degraded (7.310%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:54.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:54 smithi125 bash[77299]: cluster 2023-10-09T06:42:53.325818+0000 mgr.y (mgr.25723) 1148 : cluster 0 pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 9512/130122 objects degraded (7.310%); 14238/130122 objects misplaced (10.942%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:42:54.726 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:54 smithi125 bash[77299]: cluster 2023-10-09T06:42:54.044591+0000 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 9512/130122 objects degraded (7.310%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:54.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:54.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:54.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:54.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:54.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:54.980 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:55.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:55.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:55.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:55.147 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:55.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:55.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:55.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:55.511 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:55.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:55.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:55.769 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:55.793 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:55.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:55.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:56.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.073 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:42:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:56.112 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:42:56.160 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:42:56.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:56.203 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:56.204 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:42:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:56.244 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:42:56.283 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:42:56.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:56.315 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:42:56.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:56.358 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:42:56.361 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.362 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:42:56.442 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:56 smithi155 bash[59495]: cluster 2023-10-09T06:42:55.326935+0000 mgr.y (mgr.25723) 1149 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 9512/122790 objects degraded (7.747%); 12907/122790 objects misplaced (10.511%); 10 MiB/s, 166 objects/s recovering 2023-10-09T06:42:56.442 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:56 smithi155 bash[59495]: audit 2023-10-09T06:42:55.989210+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.125:0/3100809223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:56.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:56.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:56.578 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:56.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:56.620 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:56 smithi125 bash[80002]: cluster 2023-10-09T06:42:55.326935+0000 mgr.y (mgr.25723) 1149 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 9512/122790 objects degraded (7.747%); 12907/122790 objects misplaced (10.511%); 10 MiB/s, 166 objects/s recovering 2023-10-09T06:42:56.621 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:56 smithi125 bash[80002]: audit 2023-10-09T06:42:55.989210+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.125:0/3100809223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:56.621 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:56 smithi125 bash[77299]: cluster 2023-10-09T06:42:55.326935+0000 mgr.y (mgr.25723) 1149 : cluster 0 pgmap v1223: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 9512/122790 objects degraded (7.747%); 12907/122790 objects misplaced (10.511%); 10 MiB/s, 166 objects/s recovering 2023-10-09T06:42:56.621 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:56 smithi125 bash[77299]: audit 2023-10-09T06:42:55.989210+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.125:0/3100809223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:56.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:56.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:56.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:56.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:56.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:57.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:57.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:57.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:57.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:57.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:57.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:57.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:57.451 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:42:57.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: cluster 2023-10-09T06:42:56.420167+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: cluster 2023-10-09T06:42:56.423784+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: cluster 2023-10-09T06:42:56.425486+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: cluster 2023-10-09T06:42:56.439538+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: audit 2023-10-09T06:42:56.611115+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.125:0/3827843195' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[77299]: audit 2023-10-09T06:42:56.611962+0000 mon.a (mon.0) 1416 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.505 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: cluster 2023-10-09T06:42:56.420167+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: cluster 2023-10-09T06:42:56.423784+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:42:57.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: cluster 2023-10-09T06:42:56.425486+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: cluster 2023-10-09T06:42:56.439538+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: audit 2023-10-09T06:42:56.611115+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.125:0/3827843195' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.506 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:57 smithi125 bash[80002]: audit 2023-10-09T06:42:56.611962+0000 mon.a (mon.0) 1416 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:57.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: cluster 2023-10-09T06:42:56.420167+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: cluster 2023-10-09T06:42:56.423784+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:42:57.635 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: cluster 2023-10-09T06:42:56.425486+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: cluster 2023-10-09T06:42:56.439538+0000 osd.4 (osd.4) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-09T06:42:57.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: audit 2023-10-09T06:42:56.611115+0000 mon.c (mon.2) 109 : audit 1 from='client.? 172.21.15.125:0/3827843195' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:57 smithi155 bash[59495]: audit 2023-10-09T06:42:56.611962+0000 mon.a (mon.0) 1416 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:57.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:57.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:57.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:57.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:57.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:57.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:57.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:58.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:58.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:58.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:58.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:58.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:58.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:58.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:58.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:58.560 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:58.580 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.327410+0000 mgr.y (mgr.25723) 1150 : cluster 0 pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9512/121629 objects degraded (7.821%); 12714/121629 objects misplaced (10.453%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:58.580 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.432003+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:58.580 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: audit 2023-10-09T06:42:57.437371+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:58.580 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.437440+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:42:58.581 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.438354+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.581 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.438796+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.581 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:58 smithi155 bash[59495]: cluster 2023-10-09T06:42:57.445032+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.595 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:58.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:58.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.327410+0000 mgr.y (mgr.25723) 1150 : cluster 0 pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9512/121629 objects degraded (7.821%); 12714/121629 objects misplaced (10.453%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:58.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.432003+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:58.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: audit 2023-10-09T06:42:57.437371+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.437440+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.438354+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.438796+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[77299]: cluster 2023-10-09T06:42:57.445032+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.327410+0000 mgr.y (mgr.25723) 1150 : cluster 0 pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 79 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 9512/121629 objects degraded (7.821%); 12714/121629 objects misplaced (10.453%); 9.1 MiB/s, 144 objects/s recovering 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.432003+0000 mon.a (mon.0) 1417 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: audit 2023-10-09T06:42:57.437371+0000 mon.a (mon.0) 1418 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.437440+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:42:58.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.438354+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.438796+0000 osd.4 (osd.4) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:58 smithi125 bash[80002]: cluster 2023-10-09T06:42:57.445032+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e268 with expected crc 2023-10-09T06:42:58.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.731 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:58.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.861 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:58.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:58.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:58.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:59.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:59.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:59.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:59.262 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:59.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:59.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:59.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:59.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:59.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:59.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:59.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:42:59 smithi125 bash[77299]: cluster 2023-10-09T06:42:59.047213+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: Degraded data redundancy: 9512/121629 objects degraded (7.821%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:59.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:42:59 smithi125 bash[80002]: cluster 2023-10-09T06:42:59.047213+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: Degraded data redundancy: 9512/121629 objects degraded (7.821%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:59.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:59.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:59.789 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:42:59 smithi155 bash[59495]: cluster 2023-10-09T06:42:59.047213+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: Degraded data redundancy: 9512/121629 objects degraded (7.821%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:59.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:59.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:59.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:42:59.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:00.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:00.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:00.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:00.218 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:00.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:00.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:00.452 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:00 smithi125 bash[80002]: cluster 2023-10-09T06:42:59.327981+0000 mgr.y (mgr.25723) 1151 : cluster 0 pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 9512/119091 objects degraded (7.987%); 12450/119091 objects misplaced (10.454%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T06:43:00.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:00.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:00.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:00.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:00.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:00.748 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:00 smithi155 bash[59495]: cluster 2023-10-09T06:42:59.327981+0000 mgr.y (mgr.25723) 1151 : cluster 0 pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 9512/119091 objects degraded (7.987%); 12450/119091 objects misplaced (10.454%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T06:43:00.764 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:00.825 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:00 smithi125 bash[77299]: cluster 2023-10-09T06:42:59.327981+0000 mgr.y (mgr.25723) 1151 : cluster 0 pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 9512/119091 objects degraded (7.987%); 12450/119091 objects misplaced (10.454%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T06:43:00.867 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:00.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:00.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:01.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:01.049 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:01.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:01.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:01.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:01.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:01.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:01.542 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:43:01.548 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:43:01.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:01.577 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:01] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.33.4" 2023-10-09T06:43:01.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:01.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:01.741 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:43:01.741 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:43:01.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:01.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:01.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:01.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:01.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:02.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:02.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:02.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:02.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:02.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:02.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:02.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:02.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:02.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:43:02.582 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:02 smithi125 bash[77299]: cluster 2023-10-09T06:43:01.328682+0000 mgr.y (mgr.25723) 1152 : cluster 0 pgmap v1228: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/114630 objects degraded (8.298%); 11806/114630 objects misplaced (10.299%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:43:02.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:02 smithi125 bash[77299]: audit 2023-10-09T06:43:01.485691+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:02.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:02 smithi125 bash[80002]: cluster 2023-10-09T06:43:01.328682+0000 mgr.y (mgr.25723) 1152 : cluster 0 pgmap v1228: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/114630 objects degraded (8.298%); 11806/114630 objects misplaced (10.299%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:43:02.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:02 smithi125 bash[80002]: audit 2023-10-09T06:43:01.485691+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:02.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:02.624 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:43:02.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:02.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:02.762 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:02 smithi155 bash[59495]: cluster 2023-10-09T06:43:01.328682+0000 mgr.y (mgr.25723) 1152 : cluster 0 pgmap v1228: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/114630 objects degraded (8.298%); 11806/114630 objects misplaced (10.299%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:43:02.762 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:02 smithi155 bash[59495]: audit 2023-10-09T06:43:01.485691+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:02.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:02.881 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:02.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:02.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:02.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:03.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:03.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:03.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:03.161 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:03.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:03.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:03.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:03.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:43:03.354 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1228,"stamp":"2023-10-09T06:43:01.328170+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2507020235,"num_objects":38210,"num_object_clones":0,"num_object_copies":114630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9512,"num_objects_misplaced":11806,"num_objects_unfound":0,"num_objects_dirty":38210,"num_whiteouts":0,"num_read":6223,"num_read_kb":12659,"num_write":171982,"num_write_kb":6704952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10871,"num_bytes_recovered":701400244,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103457,"ondisk_log_size":103457,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13155532,"kb_used_data":8383700,"kb_used_omap":30,"kb_used_meta":4771745,"kb_avail":643060532,"statfs":{"total":671965249536,"available":658493984768,"internally_reserved":0,"allocated":8584908800,"data_stored":8578999200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":4886266991},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,4,6,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-338427904,"num_objects":-5164,"num_object_clones":0,"num_object_copies":-15492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2432,"num_objects_unfound":0,"num_objects_dirty":-5164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66256896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002782"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":258,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448808+0000","last_change":"2023-10-09T06:41:59.059311+0000","last_active":"2023-10-09T06:42:56.448808+0000","last_peered":"2023-10-09T06:42:56.448808+0000","last_clean":"2023-10-09T06:42:56.448808+0000","last_became_active":"2023-10-09T06:41:59.058772+0000","last_became_peered":"2023-10-09T06:41:59.058772+0000","last_unstale":"2023-10-09T06:42:56.448808+0000","last_undegraded":"2023-10-09T06:42:56.448808+0000","last_fullsized":"2023-10-09T06:42:56.448808+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":274,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.435824+0000","last_change":"2023-10-09T06:42:32.466065+0000","last_active":"2023-10-09T06:42:56.435824+0000","last_peered":"2023-10-09T06:42:56.435824+0000","last_clean":"2023-10-09T06:42:56.435824+0000","last_became_active":"2023-10-09T06:42:32.465700+0000","last_became_peered":"2023-10-09T06:42:32.465700+0000","last_unstale":"2023-10-09T06:42:56.435824+0000","last_undegraded":"2023-10-09T06:42:56.435824+0000","last_fullsized":"2023-10-09T06:42:56.435824+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:38.585481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.434063+0000","last_change":"2023-10-09T06:42:32.471750+0000","last_active":"2023-10-09T06:42:56.434063+0000","last_peered":"2023-10-09T06:42:56.434063+0000","last_clean":"2023-10-09T06:42:56.434063+0000","last_became_active":"2023-10-09T06:42:32.471408+0000","last_became_peered":"2023-10-09T06:42:32.471408+0000","last_unstale":"2023-10-09T06:42:56.434063+0000","last_undegraded":"2023-10-09T06:42:56.434063+0000","last_fullsized":"2023-10-09T06:42:56.434063+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:44.339127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":125,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451671+0000","last_change":"2023-10-09T06:38:49.383270+0000","last_active":"2023-10-09T06:42:56.451671+0000","last_peered":"2023-10-09T06:42:56.451671+0000","last_clean":"2023-10-09T06:42:56.451671+0000","last_became_active":"2023-10-09T06:38:49.382923+0000","last_became_peered":"2023-10-09T06:38:49.382923+0000","last_unstale":"2023-10-09T06:42:56.451671+0000","last_undegraded":"2023-10-09T06:42:56.451671+0000","last_fullsized":"2023-10-09T06:42:56.451671+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.434186+0000","last_change":"2023-10-09T06:41:59.061737+0000","last_active":"2023-10-09T06:42:56.434186+0000","last_peered":"2023-10-09T06:42:56.434186+0000","last_clean":"2023-10-09T06:42:56.434186+0000","last_became_active":"2023-10-09T06:41:59.061422+0000","last_became_peered":"2023-10-09T06:41:59.061422+0000","last_unstale":"2023-10-09T06:42:56.434186+0000","last_undegraded":"2023-10-09T06:42:56.434186+0000","last_fullsized":"2023-10-09T06:42:56.434186+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:09.627118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.439928+0000","last_change":"2023-10-09T06:42:32.483402+0000","last_active":"2023-10-09T06:42:56.439928+0000","last_peered":"2023-10-09T06:42:56.439928+0000","last_clean":"2023-10-09T06:42:56.439928+0000","last_became_active":"2023-10-09T06:42:32.482170+0000","last_became_peered":"2023-10-09T06:42:32.482170+0000","last_unstale":"2023-10-09T06:42:56.439928+0000","last_undegraded":"2023-10-09T06:42:56.439928+0000","last_fullsized":"2023-10-09T06:42:56.439928+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.441141+0000","last_change":"2023-10-09T06:42:32.472406+0000","last_active":"2023-10-09T06:42:56.441141+0000","last_peered":"2023-10-09T06:42:56.441141+0000","last_clean":"2023-10-09T06:42:56.441141+0000","last_became_active":"2023-10-09T06:42:32.471266+0000","last_became_peered":"2023-10-09T06:42:32.471266+0000","last_unstale":"2023-10-09T06:42:56.441141+0000","last_undegraded":"2023-10-09T06:42:56.441141+0000","last_fullsized":"2023-10-09T06:42:56.441141+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:08.342083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":306,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.434273+0000","last_change":"2023-10-09T06:42:32.482519+0000","last_active":"2023-10-09T06:42:56.434273+0000","last_peered":"2023-10-09T06:42:56.434273+0000","last_clean":"2023-10-09T06:42:56.434273+0000","last_became_active":"2023-10-09T06:42:32.482153+0000","last_became_peered":"2023-10-09T06:42:32.482153+0000","last_unstale":"2023-10-09T06:42:56.434273+0000","last_undegraded":"2023-10-09T06:42:56.434273+0000","last_fullsized":"2023-10-09T06:42:56.434273+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:32.532640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":148,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.452463+0000","last_change":"2023-10-09T06:41:59.063895+0000","last_active":"2023-10-09T06:42:56.452463+0000","last_peered":"2023-10-09T06:42:56.452463+0000","last_clean":"2023-10-09T06:42:56.452463+0000","last_became_active":"2023-10-09T06:41:59.063541+0000","last_became_peered":"2023-10-09T06:41:59.063541+0000","last_unstale":"2023-10-09T06:42:56.452463+0000","last_undegraded":"2023-10-09T06:42:56.452463+0000","last_fullsized":"2023-10-09T06:42:56.452463+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":292,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433991+0000","last_change":"2023-10-09T06:42:32.464109+0000","last_active":"2023-10-09T06:42:56.433991+0000","last_peered":"2023-10-09T06:42:56.433991+0000","last_clean":"2023-10-09T06:42:56.433991+0000","last_became_active":"2023-10-09T06:42:32.463789+0000","last_became_peered":"2023-10-09T06:42:32.463789+0000","last_unstale":"2023-10-09T06:42:56.433991+0000","last_undegraded":"2023-10-09T06:42:56.433991+0000","last_fullsized":"2023-10-09T06:42:56.433991+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:20:00.018149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":166,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451514+0000","last_change":"2023-10-09T06:19:00.254211+0000","last_active":"2023-10-09T06:42:56.451514+0000","last_peered":"2023-10-09T06:42:56.451514+0000","last_clean":"2023-10-09T06:42:56.451514+0000","last_became_active":"2023-10-09T06:18:59.810857+0000","last_became_peered":"2023-10-09T06:18:59.810857+0000","last_unstale":"2023-10-09T06:42:56.451514+0000","last_undegraded":"2023-10-09T06:42:56.451514+0000","last_fullsized":"2023-10-09T06:42:56.451514+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.435273+0000","last_change":"2023-10-09T06:42:32.482052+0000","last_active":"2023-10-09T06:42:56.435273+0000","last_peered":"2023-10-09T06:42:56.435273+0000","last_clean":"2023-10-09T06:42:56.435273+0000","last_became_active":"2023-10-09T06:42:32.481372+0000","last_became_peered":"2023-10-09T06:42:32.481372+0000","last_unstale":"2023-10-09T06:42:56.435273+0000","last_undegraded":"2023-10-09T06:42:56.435273+0000","last_fullsized":"2023-10-09T06:42:56.435273+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":354,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.452478+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:42:56.452478+0000","last_peered":"2023-10-09T06:42:56.452478+0000","last_clean":"2023-10-09T06:42:56.452478+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:42:56.452478+0000","last_undegraded":"2023-10-09T06:42:56.452478+0000","last_fullsized":"2023-10-09T06:42:56.452478+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":277,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.449169+0000","last_change":"2023-10-09T06:42:32.471553+0000","last_active":"2023-10-09T06:42:56.449169+0000","last_peered":"2023-10-09T06:42:56.449169+0000","last_clean":"2023-10-09T06:42:56.449169+0000","last_became_active":"2023-10-09T06:42:32.471184+0000","last_became_peered":"2023-10-09T06:42:32.471184+0000","last_unstale":"2023-10-09T06:42:56.449169+0000","last_undegraded":"2023-10-09T06:42:56.449169+0000","last_fullsized":"2023-10-09T06:42:56.449169+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":166,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.460669+0000","last_change":"2023-10-09T06:19:00.427970+0000","last_active":"2023-10-09T06:42:56.460669+0000","last_peered":"2023-10-09T06:42:56.460669+0000","last_clean":"2023-10-09T06:42:56.460669+0000","last_became_active":"2023-10-09T06:18:59.810949+0000","last_became_peered":"2023-10-09T06:18:59.810949+0000","last_unstale":"2023-10-09T06:42:56.460669+0000","last_undegraded":"2023-10-09T06:42:56.460669+0000","last_fullsized":"2023-10-09T06:42:56.460669+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":275,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.437305+0000","last_change":"2023-10-09T06:41:59.056318+0000","last_active":"2023-10-09T06:42:56.437305+0000","last_peered":"2023-10-09T06:42:56.437305+0000","last_clean":"2023-10-09T06:42:56.437305+0000","last_became_active":"2023-10-09T06:41:59.056072+0000","last_became_peered":"2023-10-09T06:41:59.056072+0000","last_unstale":"2023-10-09T06:42:56.437305+0000","last_undegraded":"2023-10-09T06:42:56.437305+0000","last_fullsized":"2023-10-09T06:42:56.437305+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:17.475211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.450649+0000","last_change":"2023-10-09T06:38:49.382202+0000","last_active":"2023-10-09T06:42:56.450649+0000","last_peered":"2023-10-09T06:42:56.450649+0000","last_clean":"2023-10-09T06:42:56.450649+0000","last_became_active":"2023-10-09T06:38:49.381744+0000","last_became_peered":"2023-10-09T06:38:49.381744+0000","last_unstale":"2023-10-09T06:42:56.450649+0000","last_undegraded":"2023-10-09T06:42:56.450649+0000","last_fullsized":"2023-10-09T06:42:56.450649+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451463+0000","last_change":"2023-10-09T06:42:32.491603+0000","last_active":"2023-10-09T06:42:56.451463+0000","last_peered":"2023-10-09T06:42:56.451463+0000","last_clean":"2023-10-09T06:42:56.451463+0000","last_became_active":"2023-10-09T06:42:32.491081+0000","last_became_peered":"2023-10-09T06:42:32.491081+0000","last_unstale":"2023-10-09T06:42:56.451463+0000","last_undegraded":"2023-10-09T06:42:56.451463+0000","last_fullsized":"2023-10-09T06:42:56.451463+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":205,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.450119+0000","last_change":"2023-10-09T06:42:32.471228+0000","last_active":"2023-10-09T06:42:56.450119+0000","last_peered":"2023-10-09T06:42:56.450119+0000","last_clean":"2023-10-09T06:42:56.450119+0000","last_became_active":"2023-10-09T06:42:32.469187+0000","last_became_peered":"2023-10-09T06:42:32.469187+0000","last_unstale":"2023-10-09T06:42:56.450119+0000","last_undegraded":"2023-10-09T06:42:56.450119+0000","last_fullsized":"2023-10-09T06:42:56.450119+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":117,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433964+0000","last_change":"2023-10-09T06:38:49.378852+0000","last_active":"2023-10-09T06:42:56.433964+0000","last_peered":"2023-10-09T06:42:56.433964+0000","last_clean":"2023-10-09T06:42:56.433964+0000","last_became_active":"2023-10-09T06:38:49.378523+0000","last_became_peered":"2023-10-09T06:38:49.378523+0000","last_unstale":"2023-10-09T06:42:56.433964+0000","last_undegraded":"2023-10-09T06:42:56.433964+0000","last_fullsized":"2023-10-09T06:42:56.433964+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:42.477954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":24,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.435093+0000","last_change":"2023-10-09T06:42:32.850559+0000","last_active":"2023-10-09T06:42:56.435093+0000","last_peered":"2023-10-09T06:42:56.435093+0000","last_clean":"2023-10-09T06:42:56.435093+0000","last_became_active":"2023-10-09T06:42:32.481475+0000","last_became_peered":"2023-10-09T06:42:32.481475+0000","last_unstale":"2023-10-09T06:42:56.435093+0000","last_undegraded":"2023-10-09T06:42:56.435093+0000","last_fullsized":"2023-10-09T06:42:56.435093+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":278,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.441354+0000","last_change":"2023-10-09T06:18:59.807205+0000","last_active":"2023-10-09T06:42:56.441354+0000","last_peered":"2023-10-09T06:42:56.441354+0000","last_clean":"2023-10-09T06:42:56.441354+0000","last_became_active":"2023-10-09T06:18:59.806850+0000","last_became_peered":"2023-10-09T06:18:59.806850+0000","last_unstale":"2023-10-09T06:42:56.441354+0000","last_undegraded":"2023-10-09T06:42:56.441354+0000","last_fullsized":"2023-10-09T06:42:56.441354+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.312796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"268'10700","reported_seq":10791,"reported_epoch":268,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:42:59.464196+0000","last_change":"2023-10-09T06:42:33.461630+0000","last_active":"2023-10-09T06:42:59.464196+0000","last_peered":"2023-10-09T06:42:59.464196+0000","last_clean":"2023-10-09T06:41:58.414350+0000","last_became_active":"2023-10-09T06:42:00.249637+0000","last_became_peered":"2023-10-09T06:42:00.249637+0000","last_unstale":"2023-10-09T06:42:59.464196+0000","last_undegraded":"2023-10-09T06:42:59.464196+0000","last_fullsized":"2023-10-09T06:42:59.464196+0000","mapping_epoch":258,"log_start":"248'4307","ondisk_log_start":"248'4307","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436736,"num_objects":2326,"num_object_clones":0,"num_object_copies":6978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2314,"num_objects_unfound":0,"num_objects_dirty":2326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10700,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2326}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":164,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451029+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:42:56.451029+0000","last_peered":"2023-10-09T06:42:56.451029+0000","last_clean":"2023-10-09T06:42:56.451029+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:42:56.451029+0000","last_undegraded":"2023-10-09T06:42:56.451029+0000","last_fullsized":"2023-10-09T06:42:56.451029+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.450289+0000","last_change":"2023-10-09T06:42:32.485962+0000","last_active":"2023-10-09T06:42:56.450289+0000","last_peered":"2023-10-09T06:42:56.450289+0000","last_clean":"2023-10-09T06:42:56.450289+0000","last_became_active":"2023-10-09T06:42:32.485115+0000","last_became_peered":"2023-10-09T06:42:32.485115+0000","last_unstale":"2023-10-09T06:42:56.450289+0000","last_undegraded":"2023-10-09T06:42:56.450289+0000","last_fullsized":"2023-10-09T06:42:56.450289+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"268'10804","reported_seq":10839,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:59.447189+0000","last_change":"2023-10-09T06:39:06.457503+0000","last_active":"2023-10-09T06:42:59.447189+0000","last_peered":"2023-10-09T06:42:59.447189+0000","last_clean":"2023-10-09T06:42:59.447189+0000","last_became_active":"2023-10-09T06:39:06.456829+0000","last_became_peered":"2023-10-09T06:39:06.456829+0000","last_unstale":"2023-10-09T06:42:59.447189+0000","last_undegraded":"2023-10-09T06:42:59.447189+0000","last_fullsized":"2023-10-09T06:42:59.447189+0000","mapping_epoch":241,"log_start":"247'3405","ondisk_log_start":"247'3405","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158859264,"num_objects":2424,"num_object_clones":0,"num_object_copies":7272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10804,"num_write_kb":423296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":164,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451289+0000","last_change":"2023-10-09T06:18:59.991709+0000","last_active":"2023-10-09T06:42:56.451289+0000","last_peered":"2023-10-09T06:42:56.451289+0000","last_clean":"2023-10-09T06:42:56.451289+0000","last_became_active":"2023-10-09T06:18:59.809427+0000","last_became_peered":"2023-10-09T06:18:59.809427+0000","last_unstale":"2023-10-09T06:42:56.451289+0000","last_undegraded":"2023-10-09T06:42:56.451289+0000","last_fullsized":"2023-10-09T06:42:56.451289+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448626+0000","last_change":"2023-10-09T06:42:32.476389+0000","last_active":"2023-10-09T06:42:56.448626+0000","last_peered":"2023-10-09T06:42:56.448626+0000","last_clean":"2023-10-09T06:42:56.448626+0000","last_became_active":"2023-10-09T06:42:32.475655+0000","last_became_peered":"2023-10-09T06:42:32.475655+0000","last_unstale":"2023-10-09T06:42:56.448626+0000","last_undegraded":"2023-10-09T06:42:56.448626+0000","last_fullsized":"2023-10-09T06:42:56.448626+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"268'10819","reported_seq":10854,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:59.512095+0000","last_change":"2023-10-09T06:39:06.335034+0000","last_active":"2023-10-09T06:42:59.512095+0000","last_peered":"2023-10-09T06:42:59.512095+0000","last_clean":"2023-10-09T06:42:59.512095+0000","last_became_active":"2023-10-09T06:39:06.334779+0000","last_became_peered":"2023-10-09T06:39:06.334779+0000","last_unstale":"2023-10-09T06:42:59.512095+0000","last_undegraded":"2023-10-09T06:42:59.512095+0000","last_fullsized":"2023-10-09T06:42:59.512095+0000","mapping_epoch":241,"log_start":"248'4358","ondisk_log_start":"248'4358","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152764416,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10819,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":229,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451267+0000","last_change":"2023-10-09T06:42:32.894621+0000","last_active":"2023-10-09T06:42:56.451267+0000","last_peered":"2023-10-09T06:42:56.451267+0000","last_clean":"2023-10-09T06:42:56.451267+0000","last_became_active":"2023-10-09T06:42:32.491313+0000","last_became_peered":"2023-10-09T06:42:32.491313+0000","last_unstale":"2023-10-09T06:42:56.451267+0000","last_undegraded":"2023-10-09T06:42:56.451267+0000","last_fullsized":"2023-10-09T06:42:56.451267+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.435526+0000","last_change":"2023-10-09T06:42:32.457389+0000","last_active":"2023-10-09T06:42:56.435526+0000","last_peered":"2023-10-09T06:42:56.435526+0000","last_clean":"2023-10-09T06:42:56.435526+0000","last_became_active":"2023-10-09T06:42:32.457034+0000","last_became_peered":"2023-10-09T06:42:32.457034+0000","last_unstale":"2023-10-09T06:42:56.435526+0000","last_undegraded":"2023-10-09T06:42:56.435526+0000","last_fullsized":"2023-10-09T06:42:56.435526+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"267'10375","reported_seq":15004,"reported_epoch":267,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:42:56.910174+0000","last_change":"2023-10-09T06:42:33.008590+0000","last_active":"2023-10-09T06:42:56.910174+0000","last_peered":"2023-10-09T06:42:56.910174+0000","last_clean":"2023-10-09T06:41:29.385926+0000","last_became_active":"2023-10-09T06:42:32.623940+0000","last_became_peered":"2023-10-09T06:42:32.623940+0000","last_unstale":"2023-10-09T06:42:56.910174+0000","last_undegraded":"2023-10-09T06:42:56.910174+0000","last_fullsized":"2023-10-09T06:42:32.441145+0000","mapping_epoch":263,"log_start":"247'2147","ondisk_log_start":"247'2147","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":8228,"log_dups_size":0,"ondisk_log_size":8228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:35.656779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2473,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10375,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1819,"num_bytes_recovered":119078912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,0],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"0,2","objects":6},{"shards":"1,2,6","objects":2473}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":211,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.437486+0000","last_change":"2023-10-09T06:42:32.460453+0000","last_active":"2023-10-09T06:42:56.437486+0000","last_peered":"2023-10-09T06:42:56.437486+0000","last_clean":"2023-10-09T06:42:56.437486+0000","last_became_active":"2023-10-09T06:42:32.460109+0000","last_became_peered":"2023-10-09T06:42:32.460109+0000","last_unstale":"2023-10-09T06:42:56.437486+0000","last_undegraded":"2023-10-09T06:42:56.437486+0000","last_fullsized":"2023-10-09T06:42:56.437486+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:45.978719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.459376+0000","last_change":"2023-10-09T06:13:43.430787+0000","last_active":"2023-10-09T06:42:56.459376+0000","last_peered":"2023-10-09T06:42:56.459376+0000","last_clean":"2023-10-09T06:42:56.459376+0000","last_became_active":"2023-10-09T06:13:43.430162+0000","last_became_peered":"2023-10-09T06:13:43.430162+0000","last_unstale":"2023-10-09T06:42:56.459376+0000","last_undegraded":"2023-10-09T06:42:56.459376+0000","last_fullsized":"2023-10-09T06:42:56.459376+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:26:56.992242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"268'10745","reported_seq":18090,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:58.283217+0000","last_change":"2023-10-09T06:42:33.492776+0000","last_active":"2023-10-09T06:42:58.283217+0000","last_peered":"2023-10-09T06:42:58.283217+0000","last_clean":"2023-10-09T06:42:58.283217+0000","last_became_active":"2023-10-09T06:42:33.457457+0000","last_became_peered":"2023-10-09T06:42:33.457457+0000","last_unstale":"2023-10-09T06:42:58.283217+0000","last_undegraded":"2023-10-09T06:42:58.283217+0000","last_fullsized":"2023-10-09T06:42:58.283217+0000","mapping_epoch":264,"log_start":"249'5151","ondisk_log_start":"249'5151","created":241,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:09.451383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157745152,"num_objects":2407,"num_object_clones":0,"num_object_copies":7221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10745,"num_write_kb":420864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3641,"num_bytes_recovered":238616576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":247,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433673+0000","last_change":"2023-10-09T06:14:42.680748+0000","last_active":"2023-10-09T06:42:56.433673+0000","last_peered":"2023-10-09T06:42:56.433673+0000","last_clean":"2023-10-09T06:42:56.433673+0000","last_became_active":"2023-10-09T06:14:42.680352+0000","last_became_peered":"2023-10-09T06:14:42.680352+0000","last_unstale":"2023-10-09T06:42:56.433673+0000","last_undegraded":"2023-10-09T06:42:56.433673+0000","last_fullsized":"2023-10-09T06:42:56.433673+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:33.127925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.452477+0000","last_change":"2023-10-09T06:42:32.475873+0000","last_active":"2023-10-09T06:42:56.452477+0000","last_peered":"2023-10-09T06:42:56.452477+0000","last_clean":"2023-10-09T06:42:56.452477+0000","last_became_active":"2023-10-09T06:42:32.475522+0000","last_became_peered":"2023-10-09T06:42:32.475522+0000","last_unstale":"2023-10-09T06:42:56.452477+0000","last_undegraded":"2023-10-09T06:42:56.452477+0000","last_fullsized":"2023-10-09T06:42:56.452477+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"267'10767","reported_seq":13726,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:56.815838+0000","last_change":"2023-10-09T06:42:33.734133+0000","last_active":"2023-10-09T06:42:56.815838+0000","last_peered":"2023-10-09T06:42:56.815838+0000","last_clean":"2023-10-09T06:41:29.369545+0000","last_became_active":"2023-10-09T06:42:33.623094+0000","last_became_peered":"2023-10-09T06:42:33.623094+0000","last_unstale":"2023-10-09T06:42:56.815838+0000","last_undegraded":"2023-10-09T06:42:33.613673+0000","last_fullsized":"2023-10-09T06:42:33.438677+0000","mapping_epoch":264,"log_start":"247'2442","ondisk_log_start":"247'2442","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":8325,"log_dups_size":0,"ondisk_log_size":8325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:39.563722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163512320,"num_objects":2495,"num_object_clones":0,"num_object_copies":7485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10767,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":968,"num_bytes_recovered":63045632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2,6","objects":2495},{"shards":"2,6","objects":-946}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.455313+0000","last_change":"2023-10-09T06:42:32.459309+0000","last_active":"2023-10-09T06:42:56.455313+0000","last_peered":"2023-10-09T06:42:56.455313+0000","last_clean":"2023-10-09T06:42:56.455313+0000","last_became_active":"2023-10-09T06:42:32.458936+0000","last_became_peered":"2023-10-09T06:42:32.458936+0000","last_unstale":"2023-10-09T06:42:56.455313+0000","last_undegraded":"2023-10-09T06:42:56.455313+0000","last_fullsized":"2023-10-09T06:42:56.455313+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:03.002473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448910+0000","last_change":"2023-10-09T06:14:12.891473+0000","last_active":"2023-10-09T06:42:56.448910+0000","last_peered":"2023-10-09T06:42:56.448910+0000","last_clean":"2023-10-09T06:42:56.448910+0000","last_became_active":"2023-10-09T06:14:12.890329+0000","last_became_peered":"2023-10-09T06:14:12.890329+0000","last_unstale":"2023-10-09T06:42:56.448910+0000","last_undegraded":"2023-10-09T06:42:56.448910+0000","last_fullsized":"2023-10-09T06:42:56.448910+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"268'10733","reported_seq":10783,"reported_epoch":268,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:42:59.603835+0000","last_change":"2023-10-09T06:42:32.627526+0000","last_active":"2023-10-09T06:42:59.603835+0000","last_peered":"2023-10-09T06:42:59.603835+0000","last_clean":"2023-10-09T06:41:58.402402+0000","last_became_active":"2023-10-09T06:42:32.593441+0000","last_became_peered":"2023-10-09T06:42:32.593441+0000","last_unstale":"2023-10-09T06:42:59.603835+0000","last_undegraded":"2023-10-09T06:42:59.603835+0000","last_fullsized":"2023-10-09T06:42:59.603835+0000","mapping_epoch":263,"log_start":"248'4363","ondisk_log_start":"248'4363","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6370,"log_dups_size":0,"ondisk_log_size":6370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2371,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10733,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,1],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":2371}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":251,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448475+0000","last_change":"2023-10-09T06:18:59.812160+0000","last_active":"2023-10-09T06:42:56.448475+0000","last_peered":"2023-10-09T06:42:56.448475+0000","last_clean":"2023-10-09T06:42:56.448475+0000","last_became_active":"2023-10-09T06:18:59.811660+0000","last_became_peered":"2023-10-09T06:18:59.811660+0000","last_unstale":"2023-10-09T06:42:56.448475+0000","last_undegraded":"2023-10-09T06:42:56.448475+0000","last_fullsized":"2023-10-09T06:42:56.448475+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":277,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.449775+0000","last_change":"2023-10-09T06:42:32.471822+0000","last_active":"2023-10-09T06:42:56.449775+0000","last_peered":"2023-10-09T06:42:56.449775+0000","last_clean":"2023-10-09T06:42:56.449775+0000","last_became_active":"2023-10-09T06:42:32.471090+0000","last_became_peered":"2023-10-09T06:42:32.471090+0000","last_unstale":"2023-10-09T06:42:56.449775+0000","last_undegraded":"2023-10-09T06:42:56.449775+0000","last_fullsized":"2023-10-09T06:42:56.449775+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"268'10774","reported_seq":16898,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:58.315083+0000","last_change":"2023-10-09T06:42:57.481197+0000","last_active":"2023-10-09T06:42:58.315083+0000","last_peered":"2023-10-09T06:42:58.315083+0000","last_clean":"2023-10-09T06:42:58.315083+0000","last_became_active":"2023-10-09T06:42:57.449668+0000","last_became_peered":"2023-10-09T06:42:57.449668+0000","last_unstale":"2023-10-09T06:42:58.315083+0000","last_undegraded":"2023-10-09T06:42:58.315083+0000","last_fullsized":"2023-10-09T06:42:58.315083+0000","mapping_epoch":267,"log_start":"248'5146","ondisk_log_start":"248'5146","created":241,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:30:37.637276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161873920,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10774,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3026,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":277,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.434679+0000","last_change":"2023-10-09T06:41:59.063493+0000","last_active":"2023-10-09T06:42:56.434679+0000","last_peered":"2023-10-09T06:42:56.434679+0000","last_clean":"2023-10-09T06:42:56.434679+0000","last_became_active":"2023-10-09T06:41:59.063101+0000","last_became_peered":"2023-10-09T06:41:59.063101+0000","last_unstale":"2023-10-09T06:42:56.434679+0000","last_undegraded":"2023-10-09T06:42:56.434679+0000","last_fullsized":"2023-10-09T06:42:56.434679+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:15.768257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":129,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.441211+0000","last_change":"2023-10-09T06:41:59.063964+0000","last_active":"2023-10-09T06:42:56.441211+0000","last_peered":"2023-10-09T06:42:56.441211+0000","last_clean":"2023-10-09T06:42:56.441211+0000","last_became_active":"2023-10-09T06:41:59.063716+0000","last_became_peered":"2023-10-09T06:41:59.063716+0000","last_unstale":"2023-10-09T06:42:56.441211+0000","last_undegraded":"2023-10-09T06:42:56.441211+0000","last_fullsized":"2023-10-09T06:42:56.441211+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:00.974059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"268'10606","reported_seq":10643,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:58.328636+0000","last_change":"2023-10-09T06:39:06.451992+0000","last_active":"2023-10-09T06:42:58.328636+0000","last_peered":"2023-10-09T06:42:58.328636+0000","last_clean":"2023-10-09T06:42:58.328636+0000","last_became_active":"2023-10-09T06:39:06.451648+0000","last_became_peered":"2023-10-09T06:39:06.451648+0000","last_unstale":"2023-10-09T06:42:58.328636+0000","last_undegraded":"2023-10-09T06:42:58.328636+0000","last_fullsized":"2023-10-09T06:42:58.328636+0000","mapping_epoch":241,"log_start":"248'4996","ondisk_log_start":"248'4996","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:39.385944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10606,"num_write_kb":417536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1006,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.450965+0000","last_change":"2023-10-09T06:42:33.462703+0000","last_active":"2023-10-09T06:42:56.450965+0000","last_peered":"2023-10-09T06:42:56.450965+0000","last_clean":"2023-10-09T06:42:56.450965+0000","last_became_active":"2023-10-09T06:42:33.462237+0000","last_became_peered":"2023-10-09T06:42:33.462237+0000","last_unstale":"2023-10-09T06:42:56.450965+0000","last_undegraded":"2023-10-09T06:42:56.450965+0000","last_fullsized":"2023-10-09T06:42:56.450965+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":265,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":142,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.435966+0000","last_change":"2023-10-09T06:42:32.459780+0000","last_active":"2023-10-09T06:42:56.435966+0000","last_peered":"2023-10-09T06:42:56.435966+0000","last_clean":"2023-10-09T06:42:56.435966+0000","last_became_active":"2023-10-09T06:42:32.459435+0000","last_became_peered":"2023-10-09T06:42:32.459435+0000","last_unstale":"2023-10-09T06:42:56.435966+0000","last_undegraded":"2023-10-09T06:42:56.435966+0000","last_fullsized":"2023-10-09T06:42:56.435966+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:05:58.653923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448716+0000","last_change":"2023-10-09T06:41:59.058771+0000","last_active":"2023-10-09T06:42:56.448716+0000","last_peered":"2023-10-09T06:42:56.448716+0000","last_clean":"2023-10-09T06:42:56.448716+0000","last_became_active":"2023-10-09T06:41:59.058378+0000","last_became_peered":"2023-10-09T06:41:59.058378+0000","last_unstale":"2023-10-09T06:42:56.448716+0000","last_undegraded":"2023-10-09T06:42:56.448716+0000","last_fullsized":"2023-10-09T06:42:56.448716+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.9","version":"268'10775","reported_seq":10810,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:59.589979+0000","last_change":"2023-10-09T06:39:06.331577+0000","last_active":"2023-10-09T06:42:59.589979+0000","last_peered":"2023-10-09T06:42:59.589979+0000","last_clean":"2023-10-09T06:42:59.589979+0000","last_became_active":"2023-10-09T06:39:06.330981+0000","last_became_peered":"2023-10-09T06:39:06.330981+0000","last_unstale":"2023-10-09T06:42:59.589979+0000","last_undegraded":"2023-10-09T06:42:59.589979+0000","last_fullsized":"2023-10-09T06:42:59.589979+0000","mapping_epoch":241,"log_start":"248'4483","ondisk_log_start":"248'4483","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153157632,"num_objects":2337,"num_object_clones":0,"num_object_copies":7011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10775,"num_write_kb":419584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":150,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.455446+0000","last_change":"2023-10-09T06:42:32.988904+0000","last_active":"2023-10-09T06:42:56.455446+0000","last_peered":"2023-10-09T06:42:56.455446+0000","last_clean":"2023-10-09T06:42:56.455446+0000","last_became_active":"2023-10-09T06:42:32.457840+0000","last_became_peered":"2023-10-09T06:42:32.457840+0000","last_unstale":"2023-10-09T06:42:56.455446+0000","last_undegraded":"2023-10-09T06:42:56.455446+0000","last_fullsized":"2023-10-09T06:42:56.455446+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:43.926377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":883,"num_read_kb":1177,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":278,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.449096+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:42:56.449096+0000","last_peered":"2023-10-09T06:42:56.449096+0000","last_clean":"2023-10-09T06:42:56.449096+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:42:56.449096+0000","last_undegraded":"2023-10-09T06:42:56.449096+0000","last_fullsized":"2023-10-09T06:42:56.449096+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.455943+0000","last_change":"2023-10-09T06:42:32.458547+0000","last_active":"2023-10-09T06:42:34.455943+0000","last_peered":"2023-10-09T06:42:34.455943+0000","last_clean":"2023-10-09T06:42:34.455943+0000","last_became_active":"2023-10-09T06:42:32.458191+0000","last_became_peered":"2023-10-09T06:42:32.458191+0000","last_unstale":"2023-10-09T06:42:34.455943+0000","last_undegraded":"2023-10-09T06:42:34.455943+0000","last_fullsized":"2023-10-09T06:42:34.455943+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:12.162700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"268'10639","reported_seq":10672,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:59.586209+0000","last_change":"2023-10-09T06:39:06.459488+0000","last_active":"2023-10-09T06:42:59.586209+0000","last_peered":"2023-10-09T06:42:59.586209+0000","last_clean":"2023-10-09T06:42:59.586209+0000","last_became_active":"2023-10-09T06:39:06.458713+0000","last_became_peered":"2023-10-09T06:39:06.458713+0000","last_unstale":"2023-10-09T06:42:59.586209+0000","last_undegraded":"2023-10-09T06:42:59.586209+0000","last_fullsized":"2023-10-09T06:42:59.586209+0000","mapping_epoch":241,"log_start":"248'4359","ondisk_log_start":"248'4359","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6280,"log_dups_size":0,"ondisk_log_size":6280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10639,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":635,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.455289+0000","last_change":"2023-10-09T06:14:42.682211+0000","last_active":"2023-10-09T06:42:34.455289+0000","last_peered":"2023-10-09T06:42:34.455289+0000","last_clean":"2023-10-09T06:42:34.455289+0000","last_became_active":"2023-10-09T06:14:42.681644+0000","last_became_peered":"2023-10-09T06:14:42.681644+0000","last_unstale":"2023-10-09T06:42:34.455289+0000","last_undegraded":"2023-10-09T06:42:34.455289+0000","last_fullsized":"2023-10-09T06:42:34.455289+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:09.435258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":236,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.457844+0000","last_change":"2023-10-09T06:42:32.456574+0000","last_active":"2023-10-09T06:42:34.457844+0000","last_peered":"2023-10-09T06:42:34.457844+0000","last_clean":"2023-10-09T06:42:34.457844+0000","last_became_active":"2023-10-09T06:42:32.456199+0000","last_became_peered":"2023-10-09T06:42:32.456199+0000","last_unstale":"2023-10-09T06:42:34.457844+0000","last_undegraded":"2023-10-09T06:42:34.457844+0000","last_fullsized":"2023-10-09T06:42:34.457844+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:02.675899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":112,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451254+0000","last_change":"2023-10-09T06:41:59.061063+0000","last_active":"2023-10-09T06:42:56.451254+0000","last_peered":"2023-10-09T06:42:56.451254+0000","last_clean":"2023-10-09T06:42:56.451254+0000","last_became_active":"2023-10-09T06:41:59.060687+0000","last_became_peered":"2023-10-09T06:41:59.060687+0000","last_unstale":"2023-10-09T06:42:56.451254+0000","last_undegraded":"2023-10-09T06:42:56.451254+0000","last_fullsized":"2023-10-09T06:42:56.451254+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.8","version":"268'10559","reported_seq":10594,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:58.358949+0000","last_change":"2023-10-09T06:39:06.322429+0000","last_active":"2023-10-09T06:42:58.358949+0000","last_peered":"2023-10-09T06:42:58.358949+0000","last_clean":"2023-10-09T06:42:58.358949+0000","last_became_active":"2023-10-09T06:39:06.322072+0000","last_became_peered":"2023-10-09T06:39:06.322072+0000","last_unstale":"2023-10-09T06:42:58.358949+0000","last_undegraded":"2023-10-09T06:42:58.358949+0000","last_fullsized":"2023-10-09T06:42:58.358949+0000","mapping_epoch":241,"log_start":"248'4944","ondisk_log_start":"248'4944","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:43.217621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156172288,"num_objects":2383,"num_object_clones":0,"num_object_copies":7149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10559,"num_write_kb":414144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":406,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.456214+0000","last_change":"2023-10-09T06:38:25.001852+0000","last_active":"2023-10-09T06:42:34.456214+0000","last_peered":"2023-10-09T06:42:34.456214+0000","last_clean":"2023-10-09T06:42:34.456214+0000","last_became_active":"2023-10-09T06:38:25.001121+0000","last_became_peered":"2023-10-09T06:38:25.001121+0000","last_unstale":"2023-10-09T06:42:34.456214+0000","last_undegraded":"2023-10-09T06:42:34.456214+0000","last_fullsized":"2023-10-09T06:42:34.456214+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:16.241721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":307,"num_read_kb":1996,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":563,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.449773+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:42:56.449773+0000","last_peered":"2023-10-09T06:42:56.449773+0000","last_clean":"2023-10-09T06:42:56.449773+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:42:56.449773+0000","last_undegraded":"2023-10-09T06:42:56.449773+0000","last_fullsized":"2023-10-09T06:42:56.449773+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":285,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433729+0000","last_change":"2023-10-09T06:42:32.462697+0000","last_active":"2023-10-09T06:42:56.433729+0000","last_peered":"2023-10-09T06:42:56.433729+0000","last_clean":"2023-10-09T06:42:56.433729+0000","last_became_active":"2023-10-09T06:42:32.461791+0000","last_became_peered":"2023-10-09T06:42:32.461791+0000","last_unstale":"2023-10-09T06:42:56.433729+0000","last_undegraded":"2023-10-09T06:42:56.433729+0000","last_fullsized":"2023-10-09T06:42:56.433729+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:13.161263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":241,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448992+0000","last_change":"2023-10-09T06:42:32.498269+0000","last_active":"2023-10-09T06:42:56.448992+0000","last_peered":"2023-10-09T06:42:56.448992+0000","last_clean":"2023-10-09T06:42:56.448992+0000","last_became_active":"2023-10-09T06:42:32.497812+0000","last_became_peered":"2023-10-09T06:42:32.497812+0000","last_unstale":"2023-10-09T06:42:56.448992+0000","last_undegraded":"2023-10-09T06:42:56.448992+0000","last_fullsized":"2023-10-09T06:42:56.448992+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"268'10568","reported_seq":12992,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:59.662847+0000","last_change":"2023-10-09T06:42:32.621857+0000","last_active":"2023-10-09T06:42:59.662847+0000","last_peered":"2023-10-09T06:42:59.662847+0000","last_clean":"2023-10-09T06:41:29.327663+0000","last_became_active":"2023-10-09T06:42:32.590058+0000","last_became_peered":"2023-10-09T06:42:32.590058+0000","last_unstale":"2023-10-09T06:42:59.662847+0000","last_undegraded":"2023-10-09T06:42:32.453384+0000","last_fullsized":"2023-10-09T06:42:32.438621+0000","mapping_epoch":263,"log_start":"248'4749","ondisk_log_start":"248'4749","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5819,"log_dups_size":0,"ondisk_log_size":5819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:11:34.732131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157286400,"num_objects":2400,"num_object_clones":0,"num_object_copies":7200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10568,"num_write_kb":414976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1184,"num_bytes_recovered":77529088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2400}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":224,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.450221+0000","last_change":"2023-10-09T06:41:59.618424+0000","last_active":"2023-10-09T06:42:56.450221+0000","last_peered":"2023-10-09T06:42:56.450221+0000","last_clean":"2023-10-09T06:42:56.450221+0000","last_became_active":"2023-10-09T06:41:59.091181+0000","last_became_peered":"2023-10-09T06:41:59.091181+0000","last_unstale":"2023-10-09T06:42:56.450221+0000","last_undegraded":"2023-10-09T06:42:56.450221+0000","last_fullsized":"2023-10-09T06:42:56.450221+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":229,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.456202+0000","last_change":"2023-10-09T06:14:42.679330+0000","last_active":"2023-10-09T06:42:56.456202+0000","last_peered":"2023-10-09T06:42:56.456202+0000","last_clean":"2023-10-09T06:42:56.456202+0000","last_became_active":"2023-10-09T06:14:42.679012+0000","last_became_peered":"2023-10-09T06:14:42.679012+0000","last_unstale":"2023-10-09T06:42:56.456202+0000","last_undegraded":"2023-10-09T06:42:56.456202+0000","last_fullsized":"2023-10-09T06:42:56.456202+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:48.440314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.458872+0000","last_change":"2023-10-09T06:41:59.067393+0000","last_active":"2023-10-09T06:42:56.458872+0000","last_peered":"2023-10-09T06:42:56.458872+0000","last_clean":"2023-10-09T06:42:56.458872+0000","last_became_active":"2023-10-09T06:41:59.066779+0000","last_became_peered":"2023-10-09T06:41:59.066779+0000","last_unstale":"2023-10-09T06:42:56.458872+0000","last_undegraded":"2023-10-09T06:42:56.458872+0000","last_fullsized":"2023-10-09T06:42:56.458872+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:32:35.040020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"268'10549","reported_seq":10597,"reported_epoch":268,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:42:59.565242+0000","last_change":"2023-10-09T06:42:32.490758+0000","last_active":"2023-10-09T06:42:59.565242+0000","last_peered":"2023-10-09T06:42:59.565242+0000","last_clean":"2023-10-09T06:41:58.377019+0000","last_became_active":"2023-10-09T06:42:00.376557+0000","last_became_peered":"2023-10-09T06:42:00.376557+0000","last_unstale":"2023-10-09T06:42:59.565242+0000","last_undegraded":"2023-10-09T06:42:59.565242+0000","last_fullsized":"2023-10-09T06:42:59.565242+0000","mapping_epoch":258,"log_start":"248'4226","ondisk_log_start":"248'4226","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6323,"log_dups_size":0,"ondisk_log_size":6323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153288704,"num_objects":2339,"num_object_clones":0,"num_object_copies":7017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2339,"num_objects_unfound":0,"num_objects_dirty":2339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10549,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,1],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":2339}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":178,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.451360+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:42:56.451360+0000","last_peered":"2023-10-09T06:42:56.451360+0000","last_clean":"2023-10-09T06:42:56.451360+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:42:56.451360+0000","last_undegraded":"2023-10-09T06:42:56.451360+0000","last_fullsized":"2023-10-09T06:42:56.451360+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.456044+0000","last_change":"2023-10-09T06:42:32.456666+0000","last_active":"2023-10-09T06:42:34.456044+0000","last_peered":"2023-10-09T06:42:34.456044+0000","last_clean":"2023-10-09T06:42:34.456044+0000","last_became_active":"2023-10-09T06:42:32.456305+0000","last_became_peered":"2023-10-09T06:42:32.456305+0000","last_unstale":"2023-10-09T06:42:34.456044+0000","last_undegraded":"2023-10-09T06:42:34.456044+0000","last_fullsized":"2023-10-09T06:42:34.456044+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:46.811155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":253,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.460561+0000","last_change":"2023-10-09T06:41:59.060407+0000","last_active":"2023-10-09T06:42:56.460561+0000","last_peered":"2023-10-09T06:42:56.460561+0000","last_clean":"2023-10-09T06:42:56.460561+0000","last_became_active":"2023-10-09T06:41:59.059707+0000","last_became_peered":"2023-10-09T06:41:59.059707+0000","last_unstale":"2023-10-09T06:42:56.460561+0000","last_undegraded":"2023-10-09T06:42:56.460561+0000","last_fullsized":"2023-10-09T06:42:56.460561+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"268'10693","reported_seq":10746,"reported_epoch":268,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:42:59.530994+0000","last_change":"2023-10-09T06:42:32.606657+0000","last_active":"2023-10-09T06:42:59.530994+0000","last_peered":"2023-10-09T06:42:59.530994+0000","last_clean":"2023-10-09T06:41:58.366620+0000","last_became_active":"2023-10-09T06:42:32.574285+0000","last_became_peered":"2023-10-09T06:42:32.574285+0000","last_unstale":"2023-10-09T06:42:59.530994+0000","last_undegraded":"2023-10-09T06:42:59.530994+0000","last_fullsized":"2023-10-09T06:42:59.530994+0000","mapping_epoch":263,"log_start":"248'4326","ondisk_log_start":"248'4326","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":6367,"log_dups_size":0,"ondisk_log_size":6367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151257112,"num_objects":2309,"num_object_clones":0,"num_object_copies":6927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2309,"num_objects_unfound":0,"num_objects_dirty":2309,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10693,"num_write_kb":416001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,1],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":2309}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":274,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.441287+0000","last_change":"2023-10-09T06:18:59.815973+0000","last_active":"2023-10-09T06:42:56.441287+0000","last_peered":"2023-10-09T06:42:56.441287+0000","last_clean":"2023-10-09T06:42:56.441287+0000","last_became_active":"2023-10-09T06:18:59.806999+0000","last_became_peered":"2023-10-09T06:18:59.806999+0000","last_unstale":"2023-10-09T06:42:56.441287+0000","last_undegraded":"2023-10-09T06:42:56.441287+0000","last_fullsized":"2023-10-09T06:42:56.441287+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:37.968961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":534,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:56.449736+0000","last_change":"2023-10-09T06:42:33.453416+0000","last_active":"2023-10-09T06:42:56.449736+0000","last_peered":"2023-10-09T06:42:56.449736+0000","last_clean":"2023-10-09T06:42:26.400866+0000","last_became_active":"2023-10-09T06:42:33.453013+0000","last_became_peered":"2023-10-09T06:42:33.453013+0000","last_unstale":"2023-10-09T06:42:56.449736+0000","last_undegraded":"2023-10-09T06:42:33.445382+0000","last_fullsized":"2023-10-09T06:42:33.444704+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,3],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":273,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.434088+0000","last_change":"2023-10-09T06:18:59.793344+0000","last_active":"2023-10-09T06:42:56.434088+0000","last_peered":"2023-10-09T06:42:56.434088+0000","last_clean":"2023-10-09T06:42:56.434088+0000","last_became_active":"2023-10-09T06:18:59.792898+0000","last_became_peered":"2023-10-09T06:18:59.792898+0000","last_unstale":"2023-10-09T06:42:56.434088+0000","last_undegraded":"2023-10-09T06:42:56.434088+0000","last_fullsized":"2023-10-09T06:42:56.434088+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:27.258740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"268'10770","reported_seq":10807,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-10-09T06:42:58.335128+0000","last_change":"2023-10-09T06:39:06.328261+0000","last_active":"2023-10-09T06:42:58.335128+0000","last_peered":"2023-10-09T06:42:58.335128+0000","last_clean":"2023-10-09T06:42:58.335128+0000","last_became_active":"2023-10-09T06:39:06.327897+0000","last_became_peered":"2023-10-09T06:39:06.327897+0000","last_unstale":"2023-10-09T06:42:58.335128+0000","last_undegraded":"2023-10-09T06:42:58.335128+0000","last_fullsized":"2023-10-09T06:42:58.335128+0000","mapping_epoch":241,"log_start":"249'5187","ondisk_log_start":"249'5187","created":241,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:05.296222+0000","last_clean_scrub_stamp":"2023-10-09T06:39:05.296222+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:24.948782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140672,"num_objects":2352,"num_object_clones":0,"num_object_copies":7056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10770,"num_write_kb":419904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":721,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:36.873794+0000","last_change":"2023-10-09T06:42:33.452968+0000","last_active":"2023-10-09T06:42:36.873794+0000","last_peered":"2023-10-09T06:42:36.873794+0000","last_clean":"2023-10-09T06:04:25.218264+0000","last_became_active":"2023-10-09T06:42:33.450428+0000","last_became_peered":"2023-10-09T06:42:33.450428+0000","last_unstale":"2023-10-09T06:42:36.873794+0000","last_undegraded":"2023-10-09T06:42:33.444790+0000","last_fullsized":"2023-10-09T06:42:33.443386+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:05.225120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":245,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.459768+0000","last_change":"2023-10-09T06:41:59.101497+0000","last_active":"2023-10-09T06:42:56.459768+0000","last_peered":"2023-10-09T06:42:56.459768+0000","last_clean":"2023-10-09T06:42:56.459768+0000","last_became_active":"2023-10-09T06:41:59.066320+0000","last_became_peered":"2023-10-09T06:41:59.066320+0000","last_unstale":"2023-10-09T06:42:56.459768+0000","last_undegraded":"2023-10-09T06:42:56.459768+0000","last_fullsized":"2023-10-09T06:42:56.459768+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:15.505630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448528+0000","last_change":"2023-10-09T06:42:32.476668+0000","last_active":"2023-10-09T06:42:56.448528+0000","last_peered":"2023-10-09T06:42:56.448528+0000","last_clean":"2023-10-09T06:42:56.448528+0000","last_became_active":"2023-10-09T06:42:32.476053+0000","last_became_peered":"2023-10-09T06:42:32.476053+0000","last_unstale":"2023-10-09T06:42:56.448528+0000","last_undegraded":"2023-10-09T06:42:56.448528+0000","last_fullsized":"2023-10-09T06:42:56.448528+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":233,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.459531+0000","last_change":"2023-10-09T06:18:59.800322+0000","last_active":"2023-10-09T06:42:56.459531+0000","last_peered":"2023-10-09T06:42:56.459531+0000","last_clean":"2023-10-09T06:42:56.459531+0000","last_became_active":"2023-10-09T06:18:59.799948+0000","last_became_peered":"2023-10-09T06:18:59.799948+0000","last_unstale":"2023-10-09T06:42:56.459531+0000","last_undegraded":"2023-10-09T06:42:56.459531+0000","last_fullsized":"2023-10-09T06:42:56.459531+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:59:58.858921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":238,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-10-09T06:42:34.456127+0000","last_change":"2023-10-09T06:38:22.272583+0000","last_active":"2023-10-09T06:42:34.456127+0000","last_peered":"2023-10-09T06:42:34.456127+0000","last_clean":"2023-10-09T06:42:34.456127+0000","last_became_active":"2023-10-09T06:38:22.272071+0000","last_became_peered":"2023-10-09T06:38:22.272071+0000","last_unstale":"2023-10-09T06:42:34.456127+0000","last_undegraded":"2023-10-09T06:42:34.456127+0000","last_fullsized":"2023-10-09T06:42:34.456127+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:34.640713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":101,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.452543+0000","last_change":"2023-10-09T06:42:32.593843+0000","last_active":"2023-10-09T06:42:56.452543+0000","last_peered":"2023-10-09T06:42:56.452543+0000","last_clean":"2023-10-09T06:42:56.452543+0000","last_became_active":"2023-10-09T06:42:32.483691+0000","last_became_peered":"2023-10-09T06:42:32.483691+0000","last_unstale":"2023-10-09T06:42:56.452543+0000","last_undegraded":"2023-10-09T06:42:56.452543+0000","last_fullsized":"2023-10-09T06:42:56.452543+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":256,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.449297+0000","last_change":"2023-10-09T06:41:59.060106+0000","last_active":"2023-10-09T06:42:56.449297+0000","last_peered":"2023-10-09T06:42:56.449297+0000","last_clean":"2023-10-09T06:42:56.449297+0000","last_became_active":"2023-10-09T06:41:59.059753+0000","last_became_peered":"2023-10-09T06:41:59.059753+0000","last_unstale":"2023-10-09T06:42:56.449297+0000","last_undegraded":"2023-10-09T06:42:56.449297+0000","last_fullsized":"2023-10-09T06:42:56.449297+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":88,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448861+0000","last_change":"2023-10-09T06:42:32.518723+0000","last_active":"2023-10-09T06:42:56.448861+0000","last_peered":"2023-10-09T06:42:56.448861+0000","last_clean":"2023-10-09T06:42:56.448861+0000","last_became_active":"2023-10-09T06:42:32.471529+0000","last_became_peered":"2023-10-09T06:42:32.471529+0000","last_unstale":"2023-10-09T06:42:56.448861+0000","last_undegraded":"2023-10-09T06:42:56.448861+0000","last_fullsized":"2023-10-09T06:42:56.448861+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":247,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448886+0000","last_change":"2023-10-09T06:14:42.687434+0000","last_active":"2023-10-09T06:42:56.448886+0000","last_peered":"2023-10-09T06:42:56.448886+0000","last_clean":"2023-10-09T06:42:56.448886+0000","last_became_active":"2023-10-09T06:14:42.686980+0000","last_became_peered":"2023-10-09T06:14:42.686980+0000","last_unstale":"2023-10-09T06:42:56.448886+0000","last_undegraded":"2023-10-09T06:42:56.448886+0000","last_fullsized":"2023-10-09T06:42:56.448886+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":292,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433775+0000","last_change":"2023-10-09T06:42:32.700760+0000","last_active":"2023-10-09T06:42:56.433775+0000","last_peered":"2023-10-09T06:42:56.433775+0000","last_clean":"2023-10-09T06:42:56.433775+0000","last_became_active":"2023-10-09T06:42:32.620484+0000","last_became_peered":"2023-10-09T06:42:32.620484+0000","last_unstale":"2023-10-09T06:42:56.433775+0000","last_undegraded":"2023-10-09T06:42:56.433775+0000","last_fullsized":"2023-10-09T06:42:56.433775+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:33.823549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.433922+0000","last_change":"2023-10-09T06:14:42.680070+0000","last_active":"2023-10-09T06:42:56.433922+0000","last_peered":"2023-10-09T06:42:56.433922+0000","last_clean":"2023-10-09T06:42:56.433922+0000","last_became_active":"2023-10-09T06:14:42.679487+0000","last_became_peered":"2023-10-09T06:14:42.679487+0000","last_unstale":"2023-10-09T06:42:56.433922+0000","last_undegraded":"2023-10-09T06:42:56.433922+0000","last_fullsized":"2023-10-09T06:42:56.433922+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:50.008629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":278,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.455568+0000","last_change":"2023-10-09T06:42:32.578255+0000","last_active":"2023-10-09T06:42:56.455568+0000","last_peered":"2023-10-09T06:42:56.455568+0000","last_clean":"2023-10-09T06:42:56.455568+0000","last_became_active":"2023-10-09T06:42:32.456230+0000","last_became_peered":"2023-10-09T06:42:32.456230+0000","last_unstale":"2023-10-09T06:42:56.455568+0000","last_undegraded":"2023-10-09T06:42:56.455568+0000","last_fullsized":"2023-10-09T06:42:56.455568+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:50:13.123572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":277,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-10-09T06:42:56.448588+0000","last_change":"2023-10-09T06:42:32.485746+0000","last_active":"2023-10-09T06:42:56.448588+0000","last_peered":"2023-10-09T06:42:56.448588+0000","last_clean":"2023-10-09T06:42:56.448588+0000","last_became_active":"2023-10-09T06:42:32.485341+0000","last_became_peered":"2023-10-09T06:42:32.485341+0000","last_unstale":"2023-10-09T06:42:56.448588+0000","last_undegraded":"2023-10-09T06:42:56.448588+0000","last_fullsized":"2023-10-09T06:42:56.448588+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":30632,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2502230040,"num_objects":38182,"num_object_clones":0,"num_object_copies":114546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9493,"num_objects_misplaced":11806,"num_objects_unfound":0,"num_objects_dirty":38182,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170876,"num_write_kb":6689793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10658,"num_bytes_recovered":697827328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8483713024,"data_stored":8483700808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102287,"ondisk_log_size":102287,"up":48,"acting":45,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5928,"num_read_kb":12158,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430740,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2579052,"kb_used_data":1677028,"kb_used_omap":0,"kb_used_meta":902016,"kb_avail":91166100,"statfs":{"total":95995035648,"available":93354086400,"internally_reserved":0,"allocated":1717276672,"data_stored":1716679332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923664384},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.1160000000000001,"15min":2.089},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.24099999999999999},"max":{"1min":9.4540000000000006,"5min":9.4540000000000006,"15min":178.738},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.0029999999999999,"15min":1.954},"min":{"1min":0.30199999999999999,"5min":0.27800000000000002,"15min":0.252},"max":{"1min":4.4640000000000004,"5min":5.8399999999999999,"15min":177.75},"last":1.698}]},{"osd":1,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":1.6910000000000001,"15min":1.889},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.526999999999999,"5min":13.526999999999999,"15min":148.88499999999999},"last":6.0090000000000003},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.3540000000000001,"15min":1.841},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.798999999999999,"5min":10.798999999999999,"15min":148.059},"last":0.45400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.4119999999999999,"15min":1.5089999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":5.3959999999999999,"5min":6.2670000000000003,"15min":15.957000000000001},"last":1.484},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.129,"15min":1.28},"min":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.307},"max":{"1min":3.242,"5min":5.4489999999999998,"15min":7.0629999999999997},"last":1.5129999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.0209999999999999,"15min":3.198},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":10.516999999999999,"5min":32.715000000000003,"15min":183.971},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.23,"5min":1.958,"15min":2.9540000000000002},"min":{"1min":0.47999999999999998,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":10.529999999999999,"5min":32.728999999999999,"15min":183.095},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.82499999999999996,"15min":1.036},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.28199999999999997},"max":{"1min":1.101,"5min":8.3889999999999993,"15min":12.382999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.2,"15min":1.46},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.28699999999999998},"max":{"1min":3.1819999999999999,"5min":6.0190000000000001,"15min":16.064},"last":0.73899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.05,"15min":1.1719999999999999},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.28199999999999997},"max":{"1min":1.252,"5min":5.1139999999999999,"15min":10.680999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.81899999999999995,"15min":1.2549999999999999},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.33100000000000002},"max":{"1min":1.157,"5min":1.921,"15min":11.444000000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.0820000000000001,"15min":1.2949999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.314},"max":{"1min":5.9669999999999996,"5min":5.9669999999999996,"15min":15.521000000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.2849999999999999,"15min":1.4810000000000001},"min":{"1min":0.46600000000000003,"5min":0.40400000000000003,"15min":0.378},"max":{"1min":3.0680000000000001,"5min":11.747999999999999,"15min":16.024000000000001},"last":0.77800000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626972,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1563996,"kb_used_data":978388,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92181156,"statfs":{"total":95995035648,"available":94393503744,"internally_reserved":0,"allocated":1001869312,"data_stored":1001288129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.0049999999999999,"15min":1.228},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":10.792,"5min":10.792,"15min":10.792},"last":1.946},{"interface":"front","average":{"1min":1.105,"5min":1.123,"15min":1.3580000000000001},"min":{"1min":0.38100000000000001,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":5.2770000000000001,"5min":7.0090000000000003,"15min":10.696},"last":2.7400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":1.347,"15min":2.3359999999999999},"min":{"1min":0.42199999999999999,"5min":0.33000000000000002,"15min":0.27100000000000002},"max":{"1min":7.7539999999999996,"5min":7.7539999999999996,"15min":316.88099999999997},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.2310000000000001,"15min":2.298},"min":{"1min":0.313,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.04,"5min":7.04,"15min":316.84399999999999},"last":4.1980000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:42:05 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.3500000000000001,"15min":1.8120000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.30399999999999999},"max":{"1min":7.1749999999999998,"5min":8.2390000000000008,"15min":53.616},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.149,"5min":1.6120000000000001,"15min":1.9299999999999999},"min":{"1min":0.34599999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.2030000000000003,"5min":7.9900000000000002,"15min":53.665999999999997},"last":1.96}]},{"osd":3,"last update":"Mon Oct 9 06:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":1.679,"15min":2.306},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":5.3049999999999997,"5min":22.873999999999999,"15min":115.09399999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.524,"5min":1.768,"15min":2.2109999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.32000000000000001},"max":{"1min":4.048,"5min":25.986999999999998,"15min":112.886},"last":0.51300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":5.6539999999999999,"5min":2.0030000000000001,"15min":1.478},"min":{"1min":0.373,"5min":0.35899999999999999,"15min":0.27400000000000002},"max":{"1min":80.231999999999999,"5min":80.231999999999999,"15min":80.231999999999999},"last":0.502},{"interface":"front","average":{"1min":5.6559999999999997,"5min":1.944,"15min":1.5720000000000001},"min":{"1min":0.40600000000000003,"5min":0.37,"15min":0.317},"max":{"1min":81.085999999999999,"5min":81.085999999999999,"15min":81.085999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.008,"15min":1.1339999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.35499999999999998},"max":{"1min":1.655,"5min":5.4649999999999999,"15min":10.601000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.0980000000000001,"15min":1.5009999999999999},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.29199999999999998},"max":{"1min":1.365,"5min":6.0640000000000001,"15min":12.215999999999999},"last":0.70999999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.335,"15min":1.2709999999999999},"min":{"1min":0.48299999999999998,"5min":0.432,"15min":0.35299999999999998},"max":{"1min":3.9790000000000001,"5min":7.0369999999999999,"15min":10.673},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.159,"15min":1.1679999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.33100000000000002},"max":{"1min":2.641,"5min":9.0459999999999994,"15min":9.4179999999999993},"last":0.69799999999999995}]}]},{"osd":1,"up_from":93,"seq":399431958881,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2454864,"kb_used_data":1815304,"kb_used_omap":7,"kb_used_meta":639544,"kb_avail":91290288,"statfs":{"total":95995035648,"available":93481254912,"internally_reserved":0,"allocated":1858871296,"data_stored":1857829441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":654893456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":4.7590000000000003,"5min":1.579,"15min":2.7090000000000001},"min":{"1min":0.25,"5min":0.183,"15min":0.182},"max":{"1min":17.201000000000001,"5min":17.201000000000001,"15min":132.886},"last":4.867},{"interface":"front","average":{"1min":3.4830000000000001,"5min":1.5660000000000001,"15min":2.9300000000000002},"min":{"1min":0.23200000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":15.981999999999999,"5min":15.981999999999999,"15min":133.315},"last":0.53500000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":3.3159999999999998,"5min":2.169,"15min":2.2200000000000002},"min":{"1min":0.38200000000000001,"5min":0.254,"15min":0.214},"max":{"1min":13.167999999999999,"5min":16.734000000000002,"15min":67.206000000000003},"last":0.76900000000000002},{"interface":"front","average":{"1min":4.2919999999999998,"5min":2.2879999999999998,"15min":2.3149999999999999},"min":{"1min":0.439,"5min":0.22600000000000001,"15min":0.224},"max":{"1min":13.138,"5min":16.654,"15min":67.132000000000005},"last":0.72299999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":2.6949999999999998,"15min":4.194},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":16.035,"5min":47.143000000000001,"15min":132.792},"last":2.1150000000000002},{"interface":"front","average":{"1min":2.8999999999999999,"5min":2.1230000000000002,"15min":3.996},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.24099999999999999},"max":{"1min":14.458,"5min":46.826999999999998,"15min":134.25299999999999},"last":2.0739999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.4399999999999999,"15min":1.548},"min":{"1min":0.41599999999999998,"5min":0.375,"15min":0.32600000000000001},"max":{"1min":8.0969999999999995,"5min":8.0969999999999995,"15min":26.068999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.5880000000000001,"5min":1.0449999999999999,"15min":1.2789999999999999},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":4.5679999999999996,"5min":4.5970000000000004,"15min":8.9169999999999998},"last":0.68999999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.248,"15min":1.716},"min":{"1min":0.38700000000000001,"5min":0.36699999999999999,"15min":0.35099999999999998},"max":{"1min":3.8090000000000002,"5min":5.1239999999999997,"15min":11.436999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.407,"5min":1.5700000000000001,"15min":1.625},"min":{"1min":0.47099999999999997,"5min":0.39500000000000002,"15min":0.36499999999999999},"max":{"1min":8.077,"5min":8.077,"15min":26.157},"last":0.81499999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.27,"15min":1.645},"min":{"1min":0.45900000000000002,"5min":0.36499999999999999,"15min":0.33300000000000002},"max":{"1min":3.77,"5min":5.1500000000000004,"15min":10.625},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.0990000000000002,"5min":1.6319999999999999,"15min":1.7669999999999999},"min":{"1min":0.42099999999999999,"5min":0.41399999999999998,"15min":0.34999999999999998},"max":{"1min":8.0069999999999997,"5min":8.0069999999999997,"15min":26.132000000000001},"last":0.74399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":1.7230000000000001,"15min":1.821},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.35199999999999998},"max":{"1min":8.1180000000000003,"5min":8.1180000000000003,"15min":26.109000000000002},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.7789999999999999,"5min":1.2370000000000001,"15min":1.417},"min":{"1min":0.432,"5min":0.432,"15min":0.38600000000000001},"max":{"1min":4.5949999999999998,"5min":7.8760000000000003,"15min":10.025},"last":0.64600000000000002}]}]},{"osd":0,"up_from":88,"seq":377957122407,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1279032,"kb_used_data":615728,"kb_used_omap":8,"kb_used_meta":663287,"kb_avail":92466120,"statfs":{"total":95995035648,"available":94685306880,"internally_reserved":0,"allocated":630505472,"data_stored":629491097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":679206144},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:42:36 2023","interfaces":[{"interface":"back","average":{"1min":3.1880000000000002,"5min":1.546,"15min":2.1000000000000001},"min":{"1min":0.24299999999999999,"5min":0.187,"15min":0.187},"max":{"1min":10.449,"5min":10.449,"15min":25.084},"last":0.255},{"interface":"front","average":{"1min":3.1789999999999998,"5min":1.6619999999999999,"15min":1.9570000000000001},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.20300000000000001},"max":{"1min":13.055999999999999,"5min":13.055999999999999,"15min":25.036999999999999},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:41:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.411,"15min":1.825},"min":{"1min":0.23200000000000001,"5min":0.214,"15min":0.185},"max":{"1min":8.4990000000000006,"5min":10.076000000000001,"15min":13.425000000000001},"last":0.219},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.694,"15min":1.8009999999999999},"min":{"1min":0.223,"5min":0.215,"15min":0.18099999999999999},"max":{"1min":12.448,"5min":15.351000000000001,"15min":16.129000000000001},"last":0.376}]},{"osd":3,"last update":"Mon Oct 9 06:42:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":2.617,"15min":3.0790000000000002},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":11.635999999999999,"5min":84.519999999999996,"15min":86.709999999999994},"last":4.0279999999999996},{"interface":"front","average":{"1min":3.556,"5min":2.8780000000000001,"15min":3.008},"min":{"1min":0.32700000000000001,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":15.545999999999999,"5min":87.109999999999999,"15min":87.109999999999999},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.2529999999999999,"15min":1.4570000000000001},"min":{"1min":0.35899999999999999,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":4.7560000000000002,"5min":5.5,"15min":9.4209999999999994},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.266,"15min":1.5920000000000001},"min":{"1min":0.50600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":6.3700000000000001,"5min":8.6039999999999992,"15min":10.238},"last":0.42499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":1.2589999999999999,"15min":1.7270000000000001},"min":{"1min":0.47699999999999998,"5min":0.38100000000000001,"15min":0.27800000000000002},"max":{"1min":6.4480000000000004,"5min":6.4480000000000004,"15min":12.356999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.9350000000000001,"5min":1.518,"15min":1.7390000000000001},"min":{"1min":0.41299999999999998,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":8.5630000000000006,"5min":10.286,"15min":10.419},"last":0.57799999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:42:28 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.468,"15min":1.714},"min":{"1min":0.45900000000000002,"5min":0.36799999999999999,"15min":0.33900000000000002},"max":{"1min":8.5269999999999992,"5min":8.5269999999999992,"15min":16.181999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.645,"5min":1.3420000000000001,"15min":1.498},"min":{"1min":0.42099999999999999,"5min":0.379,"15min":0.379},"max":{"1min":3.7269999999999999,"5min":6.2119999999999997,"15min":9.4510000000000005},"last":0.435}]},{"osd":7,"last update":"Mon Oct 9 06:42:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":1.5169999999999999,"15min":1.8660000000000001},"min":{"1min":0.47599999999999998,"5min":0.439,"15min":0.40300000000000002},"max":{"1min":8.4819999999999993,"5min":8.4819999999999993,"15min":13.259},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.776,"5min":1.375,"15min":1.7549999999999999},"min":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.0140000000000002,"5min":4.5960000000000001,"15min":13.428000000000001},"last":0.40799999999999997}]}]},{"osd":2,"up_from":98,"seq":420906795355,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1619952,"kb_used_data":967912,"kb_used_omap":11,"kb_used_meta":652020,"kb_avail":92125200,"statfs":{"total":95995035648,"available":94336204800,"internally_reserved":0,"allocated":991141888,"data_stored":990117982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":667669438},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8359999999999999,"5min":2.2759999999999998,"15min":2.6200000000000001},"min":{"1min":0.185,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":11.598000000000001,"5min":63.569000000000003,"15min":117.2},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.2389999999999999,"15min":2.427},"min":{"1min":0.25800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":9.8759999999999994,"5min":57.667000000000002,"15min":121.521},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":3.137,"15min":2.7069999999999999},"min":{"1min":0.223,"5min":0.20499999999999999,"15min":0.17999999999999999},"max":{"1min":11.757,"5min":138.33600000000001,"15min":138.33600000000001},"last":3.1579999999999999},{"interface":"front","average":{"1min":1.613,"5min":2.7949999999999999,"15min":2.3799999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.215},"max":{"1min":4.8499999999999996,"5min":137.96600000000001,"15min":137.96600000000001},"last":2.8530000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:42:35 2023","interfaces":[{"interface":"back","average":{"1min":4.9729999999999999,"5min":3.8119999999999998,"15min":4.2160000000000002},"min":{"1min":0.33000000000000002,"5min":0.23200000000000001,"15min":0.23000000000000001},"max":{"1min":52.060000000000002,"5min":125.762,"15min":201.33600000000001},"last":2.2440000000000002},{"interface":"front","average":{"1min":4.5129999999999999,"5min":3.5760000000000001,"15min":3.9889999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.253},"max":{"1min":51.880000000000003,"5min":125.581,"15min":199.95099999999999},"last":2.6970000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.244,"15min":1.516},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.29199999999999998},"max":{"1min":3.851,"5min":7.5590000000000002,"15min":32.433},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.333,"15min":1.4179999999999999},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.327,"5min":8.1850000000000005,"15min":32.286999999999999},"last":1.3080000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.3260000000000001,"15min":1.917},"min":{"1min":0.377,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.895,"5min":7.7199999999999998,"15min":86.423000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.4119999999999999,"15min":1.8799999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.33800000000000002},"max":{"1min":4.2750000000000004,"5min":8.2639999999999993,"15min":86.382999999999996},"last":1.3400000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.391,"15min":1.5609999999999999},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.32800000000000001},"max":{"1min":4.3609999999999998,"5min":8.2330000000000005,"15min":32.357999999999997},"last":5.3280000000000003},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.5620000000000001,"15min":1.7110000000000001},"min":{"1min":0.47699999999999998,"5min":0.376,"15min":0.35499999999999998},"max":{"1min":7.7009999999999996,"5min":11.531000000000001,"15min":32.377000000000002},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.73,"15min":1.8140000000000001},"min":{"1min":0.439,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7960000000000003,"5min":14.318,"15min":32.417999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.6839999999999999,"5min":1.518,"15min":1.6599999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.377},"max":{"1min":7.2510000000000003,"5min":8.9540000000000006,"15min":32.505000000000003},"last":0.55400000000000005}]}]},{"osd":3,"up_from":103,"seq":442381631829,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286120,"kb_used_data":1550624,"kb_used_omap":3,"kb_used_meta":735484,"kb_avail":91459032,"statfs":{"total":95995035648,"available":93654048768,"internally_reserved":0,"allocated":1587838976,"data_stored":1586777797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":753136161},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.988,"5min":2.0459999999999998,"15min":2.1480000000000001},"min":{"1min":0.20999999999999999,"5min":0.20699999999999999,"15min":0.184},"max":{"1min":9.7889999999999997,"5min":49.280000000000001,"15min":49.280000000000001},"last":1.444},{"interface":"front","average":{"1min":1.766,"5min":1.5609999999999999,"15min":1.8540000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.18099999999999999},"max":{"1min":6.5149999999999997,"5min":12.811999999999999,"15min":26.988},"last":0.374}]},{"osd":1,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":3.2429999999999999,"5min":2.7330000000000001,"15min":2.2530000000000001},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.19},"max":{"1min":10.103999999999999,"5min":69.906000000000006,"15min":69.906000000000006},"last":0.72899999999999998},{"interface":"front","average":{"1min":2.5249999999999999,"5min":2.5550000000000002,"15min":2.419},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.8119999999999994,"5min":69.855000000000004,"15min":69.855000000000004},"last":4.5670000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":2.6040000000000001,"15min":2.4649999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.20599999999999999},"max":{"1min":14.134,"5min":71.168999999999997,"15min":71.168999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.5590000000000002,"5min":2.4220000000000002,"15min":2.0619999999999998},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.23499999999999999},"max":{"1min":10.038,"5min":70.831000000000003,"15min":70.831000000000003},"last":0.79000000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":1.486,"15min":1.9299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.30399999999999999},"max":{"1min":6.5800000000000001,"5min":9.2729999999999997,"15min":72.653000000000006},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.0409999999999999,"5min":1.5469999999999999,"15min":1.819},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.29799999999999999},"max":{"1min":7.9189999999999996,"5min":9.1530000000000005,"15min":72.706999999999994},"last":0.35799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.6839999999999999,"15min":1.6679999999999999},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.35399999999999998},"max":{"1min":7.9939999999999998,"5min":9.1899999999999995,"15min":11.781000000000001},"last":0.435},{"interface":"front","average":{"1min":1.982,"5min":1.6499999999999999,"15min":1.8200000000000001},"min":{"1min":0.42499999999999999,"5min":0.39500000000000002,"15min":0.32300000000000001},"max":{"1min":6.8940000000000001,"5min":9.9329999999999998,"15min":10.315},"last":3.944}]},{"osd":6,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.3640000000000001,"15min":1.6499999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.718,"5min":9.718,"15min":15.122999999999999},"last":1.3979999999999999},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.6319999999999999,"15min":1.6930000000000001},"min":{"1min":0.48899999999999999,"5min":0.40500000000000003,"15min":0.33200000000000002},"max":{"1min":7.9669999999999996,"5min":9.109,"15min":11.728999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.8220000000000001,"15min":1.9299999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34699999999999998},"max":{"1min":6.8479999999999999,"5min":9.3469999999999995,"15min":9.3469999999999995},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.7030000000000001,"15min":1.8140000000000001},"min":{"1min":0.42299999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":9.7639999999999993,"5min":9.9550000000000001,"15min":13.211},"last":4.6020000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019438,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1372516,"kb_used_data":778716,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92372636,"statfs":{"total":95995035648,"available":94589579264,"internally_reserved":0,"allocated":797405184,"data_stored":796815422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.4350000000000001,"15min":2.1930000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.249},"max":{"1min":6.7530000000000001,"5min":61.069000000000003,"15min":115.59699999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.3260000000000001,"5min":2.4089999999999998,"15min":2.282},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28000000000000003},"max":{"1min":6.7329999999999997,"5min":61.106999999999999,"15min":115.65000000000001},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 06:42:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4930000000000001,"5min":1.304,"15min":1.3740000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28399999999999997},"max":{"1min":3.4729999999999999,"5min":6.0110000000000001,"15min":17.033000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.651,"5min":1.3720000000000001,"15min":1.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.28499999999999998},"max":{"1min":4.1139999999999999,"5min":5.992,"15min":17.015999999999998},"last":0.441}]},{"osd":2,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":2.5939999999999999,"15min":1.9970000000000001},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":108.68899999999999,"15min":108.68899999999999},"last":0.753},{"interface":"front","average":{"1min":1.8080000000000001,"5min":2.9689999999999999,"15min":2.0880000000000001},"min":{"1min":0.41399999999999998,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":7.3280000000000003,"5min":107.908,"15min":107.908},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":3.6669999999999998,"15min":3.27},"min":{"1min":0.378,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":9.7409999999999997,"5min":200.66,"15min":200.66},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.0569999999999999,"5min":3.7770000000000001,"15min":3.1709999999999998},"min":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.32300000000000001},"max":{"1min":9.7539999999999996,"5min":200.75800000000001,"15min":200.75800000000001},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.86699999999999999,"15min":1.131},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28499999999999998},"max":{"1min":0.96899999999999997,"5min":3.665,"15min":22.068000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.98399999999999999,"15min":1.2749999999999999},"min":{"1min":0.41599999999999998,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":0.96899999999999997,"5min":6.6319999999999997,"15min":13.095000000000001},"last":0.68700000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.0109999999999999,"15min":1.26},"min":{"1min":0.40999999999999998,"5min":0.39100000000000001,"15min":0.34399999999999997},"max":{"1min":1.4330000000000001,"5min":5.2649999999999997,"15min":8.1110000000000007},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.749,"5min":1.149,"15min":1.238},"min":{"1min":0.441,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":2.181,"5min":6.2320000000000002,"15min":11.782},"last":0.60699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.3100000000000001,"15min":1.385},"min":{"1min":0.39900000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.952,"5min":5.125,"15min":12.779},"last":4.2850000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.2789999999999999,"15min":1.3029999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.317},"max":{"1min":2.5870000000000002,"5min":7.835,"15min":9.1639999999999997},"last":0.71399999999999997}]}]},{"osd":5,"up_from":0,"seq":154618823205,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":614924288,"data_stored":614924288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1839534080,"data_stored":1839530008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":977076224,"data_stored":977076224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1579548672,"data_stored":1579548672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":787746816,"data_stored":787742744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":998506496,"data_stored":998506496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1686376448,"data_stored":1686372376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:43:03.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-09T06:43:03.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:03.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:03.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:03.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:03.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:03 smithi155 bash[59495]: audit 2023-10-09T06:43:01.513056+0000 mgr.y (mgr.25723) 1153 : audit 0 from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:03.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:03.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:03.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:03.818 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:03.826 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:03 smithi125 bash[77299]: audit 2023-10-09T06:43:01.513056+0000 mgr.y (mgr.25723) 1153 : audit 0 from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:03.826 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:03 smithi125 bash[80002]: audit 2023-10-09T06:43:01.513056+0000 mgr.y (mgr.25723) 1153 : audit 0 from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:03.844 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:03.952 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:03.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:03.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:04.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:04.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:04.210 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:04.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:04.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:04.376 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:04.474 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[77299]: cluster 2023-10-09T06:43:03.338974+0000 mgr.y (mgr.25723) 1154 : cluster 0 pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9512/113499 objects degraded (8.381%); 11618/113499 objects misplaced (10.236%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:43:04.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:04.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:04.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:04.645 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:04.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:04.756 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:04 smithi155 bash[59495]: cluster 2023-10-09T06:43:03.338974+0000 mgr.y (mgr.25723) 1154 : cluster 0 pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9512/113499 objects degraded (8.381%); 11618/113499 objects misplaced (10.236%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:43:04.756 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:04 smithi155 bash[59495]: audit 2023-10-09T06:43:03.603044+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:04.756 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:04 smithi155 bash[59495]: cluster 2023-10-09T06:43:04.048368+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 9512/113499 objects degraded (8.381%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:04.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:04.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:04.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[77299]: audit 2023-10-09T06:43:03.603044+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[77299]: cluster 2023-10-09T06:43:04.048368+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 9512/113499 objects degraded (8.381%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[80002]: cluster 2023-10-09T06:43:03.338974+0000 mgr.y (mgr.25723) 1154 : cluster 0 pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 9512/113499 objects degraded (8.381%); 11618/113499 objects misplaced (10.236%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:43:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[80002]: audit 2023-10-09T06:43:03.603044+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:04.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:04 smithi125 bash[80002]: cluster 2023-10-09T06:43:04.048368+0000 mon.a (mon.0) 1423 : cluster 3 Health check update: Degraded data redundancy: 9512/113499 objects degraded (8.381%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:04.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:04.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:05.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:05.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:05.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:05.162 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:05.265 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:05.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:05.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:05.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:05.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:05.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:05.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:05.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.834 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:05.938 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:05.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:06.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:06.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:06.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:06.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:06.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:06.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:06.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:06.475 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:06.482 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:06.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:06.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:06.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:06 smithi155 bash[59495]: cluster 2023-10-09T06:43:05.339791+0000 mgr.y (mgr.25723) 1155 : cluster 0 pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 9512/105558 objects degraded (9.011%); 10868/105558 objects misplaced (10.296%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:43:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:06.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:06.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:06.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:06.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:06 smithi125 bash[77299]: cluster 2023-10-09T06:43:05.339791+0000 mgr.y (mgr.25723) 1155 : cluster 0 pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 9512/105558 objects degraded (9.011%); 10868/105558 objects misplaced (10.296%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:43:06.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:06 smithi125 bash[80002]: cluster 2023-10-09T06:43:05.339791+0000 mgr.y (mgr.25723) 1155 : cluster 0 pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 9512/105558 objects degraded (9.011%); 10868/105558 objects misplaced (10.296%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:43:06.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:06.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:06.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:06.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:07.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:07.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:07.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:07.204 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:07.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:07.337 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:07.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:07.504 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:43:07.507 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:07.507 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:07.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:07.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:07.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:07.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:07.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:07.803 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:07.913 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:07.939 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:07.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:08.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:08.168 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:08.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:08.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.323 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:08.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.455 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:08.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:08.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[77299]: cluster 2023-10-09T06:43:07.340347+0000 mgr.y (mgr.25723) 1156 : cluster 0 pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9512/104394 objects degraded (9.112%); 10667/104394 objects misplaced (10.218%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[77299]: audit 2023-10-09T06:43:07.910973+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[77299]: audit 2023-10-09T06:43:07.925121+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[77299]: audit 2023-10-09T06:43:08.260442+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[77299]: audit 2023-10-09T06:43:08.266926+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[80002]: cluster 2023-10-09T06:43:07.340347+0000 mgr.y (mgr.25723) 1156 : cluster 0 pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9512/104394 objects degraded (9.112%); 10667/104394 objects misplaced (10.218%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T06:43:08.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[80002]: audit 2023-10-09T06:43:07.910973+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[80002]: audit 2023-10-09T06:43:07.925121+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[80002]: audit 2023-10-09T06:43:08.260442+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:08 smithi125 bash[80002]: audit 2023-10-09T06:43:08.266926+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:08.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:08.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:08.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:08 smithi155 bash[59495]: cluster 2023-10-09T06:43:07.340347+0000 mgr.y (mgr.25723) 1156 : cluster 0 pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 9512/104394 objects degraded (9.112%); 10667/104394 objects misplaced (10.218%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T06:43:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:08 smithi155 bash[59495]: audit 2023-10-09T06:43:07.910973+0000 mon.a (mon.0) 1424 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:08 smithi155 bash[59495]: audit 2023-10-09T06:43:07.925121+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:08 smithi155 bash[59495]: audit 2023-10-09T06:43:08.260442+0000 mon.a (mon.0) 1426 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:08 smithi155 bash[59495]: audit 2023-10-09T06:43:08.266926+0000 mon.a (mon.0) 1427 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:08.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:08.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:08.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:09.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:09.123 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:09.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:09.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:09.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:09.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:09 smithi155 bash[59495]: audit 2023-10-09T06:43:08.741548+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:43:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:09 smithi155 bash[59495]: audit 2023-10-09T06:43:08.743880+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:43:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:09 smithi155 bash[59495]: audit 2023-10-09T06:43:08.753953+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:09 smithi155 bash[59495]: cluster 2023-10-09T06:43:09.051618+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 9512/104394 objects degraded (9.112%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:09.489 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:09 smithi155 bash[59495]: cluster 2023-10-09T06:43:09.340964+0000 mgr.y (mgr.25723) 1157 : cluster 0 pgmap v1232: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/101595 objects degraded (9.363%); 10667/101595 objects misplaced (10.500%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:43:09.501 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:09.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[77299]: audit 2023-10-09T06:43:08.741548+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[77299]: audit 2023-10-09T06:43:08.743880+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[77299]: audit 2023-10-09T06:43:08.753953+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[77299]: cluster 2023-10-09T06:43:09.051618+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 9512/104394 objects degraded (9.112%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[77299]: cluster 2023-10-09T06:43:09.340964+0000 mgr.y (mgr.25723) 1157 : cluster 0 pgmap v1232: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/101595 objects degraded (9.363%); 10667/101595 objects misplaced (10.500%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[80002]: audit 2023-10-09T06:43:08.741548+0000 mon.a (mon.0) 1428 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:43:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[80002]: audit 2023-10-09T06:43:08.743880+0000 mon.a (mon.0) 1429 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:43:09.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[80002]: audit 2023-10-09T06:43:08.753953+0000 mon.a (mon.0) 1430 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:43:09.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[80002]: cluster 2023-10-09T06:43:09.051618+0000 mon.a (mon.0) 1431 : cluster 3 Health check update: Degraded data redundancy: 9512/104394 objects degraded (9.112%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:09.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:09 smithi125 bash[80002]: cluster 2023-10-09T06:43:09.340964+0000 mgr.y (mgr.25723) 1157 : cluster 0 pgmap v1232: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/101595 objects degraded (9.363%); 10667/101595 objects misplaced (10.500%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:43:09.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:09.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.812 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:09.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:09.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:09.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:09.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:10.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:10.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:10.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:10.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:10.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:10.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:10.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:10.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:10.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:10.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:10.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:10.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:10.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:10.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:10.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:10.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:10.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:10.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:11.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:11.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:11.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:11.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:11.258 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:11] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.33.4" 2023-10-09T06:43:11.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:11.390 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.415 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:11.522 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:11.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:11.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:11.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:11.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:11.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:11.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:12.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:12.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:12.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:12.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:12.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:12.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:12.613 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:12.621 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:12 smithi155 bash[59495]: cluster 2023-10-09T06:43:11.341667+0000 mgr.y (mgr.25723) 1158 : cluster 0 pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9512/96843 objects degraded (9.822%); 9948/96843 objects misplaced (10.272%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:12.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:12.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:12 smithi125 bash[77299]: cluster 2023-10-09T06:43:11.341667+0000 mgr.y (mgr.25723) 1158 : cluster 0 pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9512/96843 objects degraded (9.822%); 9948/96843 objects misplaced (10.272%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:12.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:12 smithi125 bash[80002]: cluster 2023-10-09T06:43:11.341667+0000 mgr.y (mgr.25723) 1158 : cluster 0 pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 9512/96843 objects degraded (9.822%); 9948/96843 objects misplaced (10.272%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:12.712 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:43:12.712 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:43:12.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:12.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:12.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:12.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:12.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:12.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:13.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:13.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:13.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:13.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:13.280 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:13.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:13.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:13.445 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:13.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:13.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:13.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:13.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:13.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:13.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:13.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:13.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:43:13.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:13.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:13.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:14.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:14.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:14.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:14.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:14.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:14.529 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:14.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:14 smithi125 bash[77299]: cluster 2023-10-09T06:43:13.342240+0000 mgr.y (mgr.25723) 1159 : cluster 0 pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 9512/95448 objects degraded (9.966%); 9762/95448 objects misplaced (10.228%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:43:14.535 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:14 smithi125 bash[77299]: cluster 2023-10-09T06:43:14.052839+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: Degraded data redundancy: 9512/95448 objects degraded (9.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:14.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:14 smithi125 bash[80002]: cluster 2023-10-09T06:43:13.342240+0000 mgr.y (mgr.25723) 1159 : cluster 0 pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 9512/95448 objects degraded (9.966%); 9762/95448 objects misplaced (10.228%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:43:14.535 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:14 smithi125 bash[80002]: cluster 2023-10-09T06:43:14.052839+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: Degraded data redundancy: 9512/95448 objects degraded (9.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:14.636 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.662 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:14.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.793 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:14.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:14 smithi155 bash[59495]: cluster 2023-10-09T06:43:13.342240+0000 mgr.y (mgr.25723) 1159 : cluster 0 pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 9512/95448 objects degraded (9.966%); 9762/95448 objects misplaced (10.228%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:43:14.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:14 smithi155 bash[59495]: cluster 2023-10-09T06:43:14.052839+0000 mon.a (mon.0) 1432 : cluster 3 Health check update: Degraded data redundancy: 9512/95448 objects degraded (9.966%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:14.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:14.922 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:14.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:15.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:15.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:15.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:15.190 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:15.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:15.322 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:15.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:15.451 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:15.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:15.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:15.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:15.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:15.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:15.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:15.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:15.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:15.995 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:16.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:16.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:16.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:16.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:16.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:16.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:16 smithi125 bash[77299]: cluster 2023-10-09T06:43:15.343017+0000 mgr.y (mgr.25723) 1160 : cluster 0 pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 9512/87861 objects degraded (10.826%); 8988/87861 objects misplaced (10.230%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:16.446 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:16 smithi125 bash[80002]: cluster 2023-10-09T06:43:15.343017+0000 mgr.y (mgr.25723) 1160 : cluster 0 pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 9512/87861 objects degraded (10.826%); 8988/87861 objects misplaced (10.230%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:16.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:16.524 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:16 smithi155 bash[59495]: cluster 2023-10-09T06:43:15.343017+0000 mgr.y (mgr.25723) 1160 : cluster 0 pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 711 op/s; 9512/87861 objects degraded (10.826%); 8988/87861 objects misplaced (10.230%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:16.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:16.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:16.808 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:16.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:16.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:16.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:16.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:16.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:17.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:17.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:17.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:17.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:17.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:17.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:17.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:17.615 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:17.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:17.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:17.773 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:17.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:17.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:17.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:18.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:18.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:18.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:18.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:18.550 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.561 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:18 smithi155 bash[59495]: cluster 2023-10-09T06:43:17.343573+0000 mgr.y (mgr.25723) 1161 : cluster 0 pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9512/86646 objects degraded (10.978%); 8783/86646 objects misplaced (10.137%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:43:18.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:18.648 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:43:18.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:18.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:18.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:18.734 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:18 smithi125 bash[77299]: cluster 2023-10-09T06:43:17.343573+0000 mgr.y (mgr.25723) 1161 : cluster 0 pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9512/86646 objects degraded (10.978%); 8783/86646 objects misplaced (10.137%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:43:18.735 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:18 smithi125 bash[80002]: cluster 2023-10-09T06:43:17.343573+0000 mgr.y (mgr.25723) 1161 : cluster 0 pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9512/86646 objects degraded (10.978%); 8783/86646 objects misplaced (10.137%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:43:18.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:18.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:18.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:18.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:19.081 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:19.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:19.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:19.243 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:19.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:19.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:19.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:19.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:19.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:19.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:19.624 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:19 smithi125 bash[77299]: audit 2023-10-09T06:43:18.603282+0000 mon.a (mon.0) 1433 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:19.624 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:19 smithi125 bash[77299]: cluster 2023-10-09T06:43:19.053925+0000 mon.a (mon.0) 1434 : cluster 3 Health check update: Degraded data redundancy: 9512/86646 objects degraded (10.978%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:19.624 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:19 smithi125 bash[80002]: audit 2023-10-09T06:43:18.603282+0000 mon.a (mon.0) 1433 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:19.624 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:19 smithi125 bash[80002]: cluster 2023-10-09T06:43:19.053925+0000 mon.a (mon.0) 1434 : cluster 3 Health check update: Degraded data redundancy: 9512/86646 objects degraded (10.978%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:19.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:19.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:19.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:19.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:19 smithi155 bash[59495]: audit 2023-10-09T06:43:18.603282+0000 mon.a (mon.0) 1433 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:19 smithi155 bash[59495]: cluster 2023-10-09T06:43:19.053925+0000 mon.a (mon.0) 1434 : cluster 3 Health check update: Degraded data redundancy: 9512/86646 objects degraded (10.978%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:19.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:19.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:20.018 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:20.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:20.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:20.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:20.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:20.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:20.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:20.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.569 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:20.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:20.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:20.681 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:20 smithi125 bash[80002]: cluster 2023-10-09T06:43:19.344187+0000 mgr.y (mgr.25723) 1162 : cluster 0 pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 9512/82776 objects degraded (11.491%); 8783/82776 objects misplaced (10.611%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:20.681 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:20 smithi125 bash[77299]: cluster 2023-10-09T06:43:19.344187+0000 mgr.y (mgr.25723) 1162 : cluster 0 pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 9512/82776 objects degraded (11.491%); 8783/82776 objects misplaced (10.611%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:20.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:20.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:20.826 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:20 smithi155 bash[59495]: cluster 2023-10-09T06:43:19.344187+0000 mgr.y (mgr.25723) 1162 : cluster 0 pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 9512/82776 objects degraded (11.491%); 8783/82776 objects misplaced (10.611%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:20.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:20.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:20.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:20.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:20.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:21.073 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:21.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:21.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:21.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:21.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:21.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:21.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:21.492 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:21] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T06:43:21.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:21.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:21.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:21.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:21.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:21.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:21.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:21.866 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:43:21.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:21.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:21.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:21.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:21.918 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:43:21.981 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:43:22.009 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:22.023 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:43:22.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:22.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:22.069 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:43:22.107 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:43:22.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:22.156 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:43:22.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:22.198 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.207 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:22.302 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:22.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:22.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:22.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:22.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:22.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:22.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:22.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:22 smithi125 bash[77299]: cluster 2023-10-09T06:43:21.345630+0000 mgr.y (mgr.25723) 1163 : cluster 0 pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9512/77745 objects degraded (12.235%); 7977/77745 objects misplaced (10.260%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:22.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:22 smithi125 bash[77299]: audit 2023-10-09T06:43:21.771555+0000 mon.a (mon.0) 1435 : audit 0 from='client.? 172.21.15.125:0/2534540735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:22 smithi125 bash[80002]: cluster 2023-10-09T06:43:21.345630+0000 mgr.y (mgr.25723) 1163 : cluster 0 pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9512/77745 objects degraded (12.235%); 7977/77745 objects misplaced (10.260%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:22.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:22 smithi125 bash[80002]: audit 2023-10-09T06:43:21.771555+0000 mon.a (mon.0) 1435 : audit 0 from='client.? 172.21.15.125:0/2534540735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:22.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:22.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:22.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:22 smithi155 bash[59495]: cluster 2023-10-09T06:43:21.345630+0000 mgr.y (mgr.25723) 1163 : cluster 0 pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 9512/77745 objects degraded (12.235%); 7977/77745 objects misplaced (10.260%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:22 smithi155 bash[59495]: audit 2023-10-09T06:43:21.771555+0000 mon.a (mon.0) 1435 : audit 0 from='client.? 172.21.15.125:0/2534540735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:22.980 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:22.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:23.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:23.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:23.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:23.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:23.342 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:23.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:23.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:23.452 INFO:teuthology.orchestra.run.smithi125.stderr:set pool 9 pgp_num to 16 2023-10-09T06:43:23.453 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:43:23.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:23.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:23.605 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:23.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:23.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[77299]: audit 2023-10-09T06:43:22.517963+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.125:0/1960071364' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[77299]: audit 2023-10-09T06:43:22.520003+0000 mon.a (mon.0) 1436 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[77299]: audit 2023-10-09T06:43:22.541434+0000 mon.a (mon.0) 1437 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:23.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[80002]: audit 2023-10-09T06:43:22.517963+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.125:0/1960071364' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[80002]: audit 2023-10-09T06:43:22.520003+0000 mon.a (mon.0) 1436 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:23 smithi125 bash[80002]: audit 2023-10-09T06:43:22.541434+0000 mon.a (mon.0) 1437 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:23.728 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:23 smithi155 bash[59495]: audit 2023-10-09T06:43:22.517963+0000 mon.c (mon.2) 110 : audit 1 from='client.? 172.21.15.125:0/1960071364' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.728 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:23 smithi155 bash[59495]: audit 2023-10-09T06:43:22.520003+0000 mon.a (mon.0) 1436 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:43:23.728 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:23 smithi155 bash[59495]: audit 2023-10-09T06:43:22.541434+0000 mon.a (mon.0) 1437 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:23.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:23.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:23.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:23.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:23.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:23.994 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:24.022 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:24.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186570 9.5 9:a24cd665:::benchmark_data_smithi125_199533_object82039:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.131995+0000", 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.023566237, 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023618904, 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:43:24.159 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186570 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.131995+0000", 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.131995+0000", 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.131994+0000", 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5059999999999997e-06 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.131999+0000", 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9819999999999997e-06 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.132008+0000", 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5809999999999999e-06 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.132015+0000", 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000421734 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.132437+0000", 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109175 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.132546+0000", 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9322999999999998e-05 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1,3", 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.132645+0000", 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091264689999999999 2023-10-09T06:43:24.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.141772+0000", 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001132765 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.142905+0000", 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012268677 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.155173+0000", 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00041490400000000001 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.155588+0000", 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.7754000000000003e-05 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:24.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.155646+0000", 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.023651052999999998 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186574 9.e 9:726e7f12:::benchmark_data_smithi125_199533_object82043:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.145662+0000", 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0098995420000000008, 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010209706000000001, 2023-10-09T06:43:24.167 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186574 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.168 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145662+0000", 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145662+0000", 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145660+0000", 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2899999999999999e-06 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145666+0000", 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6947000000000001e-05 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145693+0000", 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2420000000000003e-06 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145701+0000", 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5214000000000003e-05 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145766+0000", 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018582900000000001 2023-10-09T06:43:24.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145952+0000", 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021890000000000001 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:43:24.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.146171+0000", 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042127850000000001 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.150383+0000", 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0047217140000000001 2023-10-09T06:43:24.173 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186577 9.5 9:a46650f5:::benchmark_data_smithi125_199533_object82046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.145718+0000", 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0098436670000000004, 2023-10-09T06:43:24.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010266456, 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186577 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.175 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145718+0000", 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145718+0000", 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145717+0000", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.019e-06 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145721+0000", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4260000000000003e-06 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145729+0000", 2023-10-09T06:43:24.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0869999999999997e-06 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.145734+0000", 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0051015870000000003 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.150836+0000", 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016440000000000001 2023-10-09T06:43:24.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.151000+0000", 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017217999999999999 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1,3", 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.151173+0000", 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041651320000000002 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.155338+0000", 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096201959999999993 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186578 9.e 9:78a4ff7a:::benchmark_data_smithi125_199533_object82047:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.147009+0000", 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.0085517630000000004, 2023-10-09T06:43:24.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0090744970000000008, 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186578 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147009+0000", 2023-10-09T06:43:24.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147009+0000", 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147008+0000", 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4829999999999996e-06 2023-10-09T06:43:24.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147015+0000", 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5574e-05 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147030+0000", 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7109999999999995e-06 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.147038+0000", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033772889999999999 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.150415+0000", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000171538 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.150587+0000", 2023-10-09T06:43:24.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000223334 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.150810+0000", 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038008489999999998 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.185 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186580 9.a 9:5ec9cec1:::benchmark_data_smithi125_199533_object82049:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.153711+0000", 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001850584, 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0024632080000000002, 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "started", 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186580 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.186 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153711+0000", 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153711+0000", 2023-10-09T06:43:24.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153709+0000", 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4949999999999996e-06 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153716+0000", 2023-10-09T06:43:24.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8679999999999994e-06 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153726+0000", 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1340000000000003e-06 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153731+0000", 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7002999999999995e-05 2023-10-09T06:43:24.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153798+0000", 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016564800000000001 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153963+0000", 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025277399999999999 2023-10-09T06:43:24.190 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:186583 9.9 9:94cc7311:::benchmark_data_smithi125_199533_object82052:head [delete] snapc 0=[] ondisk+write+known_if_redirected e269)", 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:43:24.153744+0000", 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 0.001817096, 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002531302, 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:24.191 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 186583 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153744+0000", 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:24.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153744+0000", 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153744+0000", 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6619999999999998e-06 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153747+0000", 2023-10-09T06:43:24.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0779999999999998e-06 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153754+0000", 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7480000000000001e-06 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153759+0000", 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1876999999999998e-05 2023-10-09T06:43:24.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153811+0000", 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119869 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.153931+0000", 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000176241 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:24.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:43:24.154107+0000", 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036288800000000001 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: ], 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 6 2023-10-09T06:43:24.196 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:43:24.257 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:24.288 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:24.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:24.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:24.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:24.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:23.346220+0000 mgr.y (mgr.25723) 1164 : cluster 0 pgmap v1239: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 9512/75855 objects degraded (12.540%); 7780/75855 objects misplaced (10.256%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:23.426059+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: audit 2023-10-09T06:43:23.439966+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: audit 2023-10-09T06:43:23.440180+0000 mon.a (mon.0) 1440 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:23.440353+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:43:24.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:23.440946+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:23.441295+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[77299]: cluster 2023-10-09T06:43:24.055057+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: Degraded data redundancy: 9512/75855 objects degraded (12.540%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:23.346220+0000 mgr.y (mgr.25723) 1164 : cluster 0 pgmap v1239: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 9512/75855 objects degraded (12.540%); 7780/75855 objects misplaced (10.256%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:23.426059+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: audit 2023-10-09T06:43:23.439966+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: audit 2023-10-09T06:43:23.440180+0000 mon.a (mon.0) 1440 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:24.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:23.440353+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:43:24.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:23.440946+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:23.441295+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.641 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:24 smithi125 bash[80002]: cluster 2023-10-09T06:43:24.055057+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: Degraded data redundancy: 9512/75855 objects degraded (12.540%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:23.346220+0000 mgr.y (mgr.25723) 1164 : cluster 0 pgmap v1239: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 9512/75855 objects degraded (12.540%); 7780/75855 objects misplaced (10.256%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:23.426059+0000 mon.a (mon.0) 1438 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: audit 2023-10-09T06:43:23.439966+0000 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: audit 2023-10-09T06:43:23.440180+0000 mon.a (mon.0) 1440 : audit 1 from='client.? 172.21.15.125:0/3361898769' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:23.440353+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:23.440946+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:23.441295+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e269 with expected crc 2023-10-09T06:43:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:24 smithi155 bash[59495]: cluster 2023-10-09T06:43:24.055057+0000 mon.a (mon.0) 1442 : cluster 3 Health check update: Degraded data redundancy: 9512/75855 objects degraded (12.540%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:24.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:24.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:24.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:24.793 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:24.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:24.956 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:24.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:25.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:25.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:25.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:25.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:25.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:43:25.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:25.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:25.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:25.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:25.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:25.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:25.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:25.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:25.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:25.993 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:26.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:26.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.189 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:26.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:26.322 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:43:26.332 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:26.435 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.495 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:26.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:26.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:26.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:26.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:26 smithi125 bash[77299]: cluster 2023-10-09T06:43:25.347389+0000 mgr.y (mgr.25723) 1165 : cluster 0 pgmap v1241: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9512/68217 objects degraded (13.944%); 6992/68217 objects misplaced (10.250%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:43:26.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:26 smithi125 bash[80002]: cluster 2023-10-09T06:43:25.347389+0000 mgr.y (mgr.25723) 1165 : cluster 0 pgmap v1241: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9512/68217 objects degraded (13.944%); 6992/68217 objects misplaced (10.250%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:43:26.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:26.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:26.881 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:26 smithi155 bash[59495]: cluster 2023-10-09T06:43:25.347389+0000 mgr.y (mgr.25723) 1165 : cluster 0 pgmap v1241: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 9512/68217 objects degraded (13.944%); 6992/68217 objects misplaced (10.250%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:43:26.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:26.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:27.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:27.044 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:27.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:27.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:27.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:27.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:27.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:27.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:27.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:27.603 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:27.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:27.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:27.742 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:27.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:27.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:27.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:27.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:28.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:28.155 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:28.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:28.297 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:28.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:28.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:28.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:28.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:43:28.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:28.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:28.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:28.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:28 smithi125 bash[77299]: cluster 2023-10-09T06:43:27.347963+0000 mgr.y (mgr.25723) 1166 : cluster 0 pgmap v1242: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9512/66984 objects degraded (14.200%); 6796/66984 objects misplaced (10.146%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:28.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:28 smithi125 bash[80002]: cluster 2023-10-09T06:43:27.347963+0000 mgr.y (mgr.25723) 1166 : cluster 0 pgmap v1242: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9512/66984 objects degraded (14.200%); 6796/66984 objects misplaced (10.146%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:28.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:28.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:28.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:28 smithi155 bash[59495]: cluster 2023-10-09T06:43:27.347963+0000 mgr.y (mgr.25723) 1166 : cluster 0 pgmap v1242: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 9512/66984 objects degraded (14.200%); 6796/66984 objects misplaced (10.146%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:28.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:28.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.977 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:28.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:29.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:29.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:29.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:29.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:29.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:29.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:29.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:29.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:29.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:29.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:29 smithi125 bash[77299]: cluster 2023-10-09T06:43:29.059083+0000 mon.a (mon.0) 1443 : cluster 3 Health check update: Degraded data redundancy: 9512/66984 objects degraded (14.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:29 smithi125 bash[80002]: cluster 2023-10-09T06:43:29.059083+0000 mon.a (mon.0) 1443 : cluster 3 Health check update: Degraded data redundancy: 9512/66984 objects degraded (14.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:29.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:29 smithi155 bash[59495]: cluster 2023-10-09T06:43:29.059083+0000 mon.a (mon.0) 1443 : cluster 3 Health check update: Degraded data redundancy: 9512/66984 objects degraded (14.200%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:29.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:29.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:29.794 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:29.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:29.927 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:29.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:30.033 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:30.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:30.091 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:43:30.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:30.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:30.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:30.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:30.464 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:30 smithi125 bash[77299]: cluster 2023-10-09T06:43:29.348629+0000 mgr.y (mgr.25723) 1167 : cluster 0 pgmap v1243: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 9512/63234 objects degraded (15.043%); 6796/63234 objects misplaced (10.747%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:30.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:30.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:30 smithi155 bash[59495]: cluster 2023-10-09T06:43:29.348629+0000 mgr.y (mgr.25723) 1167 : cluster 0 pgmap v1243: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 9512/63234 objects degraded (15.043%); 6796/63234 objects misplaced (10.747%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:30.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:30.743 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:30.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:30 smithi125 bash[80002]: cluster 2023-10-09T06:43:29.348629+0000 mgr.y (mgr.25723) 1167 : cluster 0 pgmap v1243: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 9512/63234 objects degraded (15.043%); 6796/63234 objects misplaced (10.747%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:30.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:30.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:30.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:31.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:31.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:31.174 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:31.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:31.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:31] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T06:43:31.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:31.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:31.480 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:43:31.480 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:43:31.480 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 7, 6] 2023-10-09T06:43:31.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T06:43:31.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:31.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:31.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:31.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:31.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:31.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:31.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:31.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:32.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:32.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:32.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:32.285 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:32.390 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:32.423 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:32.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:32.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:32.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:32.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:32.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:32.783 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:32 smithi125 bash[77299]: cluster 2023-10-09T06:43:31.349321+0000 mgr.y (mgr.25723) 1168 : cluster 0 pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/58362 objects degraded (16.298%); 5970/58362 objects misplaced (10.229%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:32.791 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:32 smithi125 bash[80002]: cluster 2023-10-09T06:43:31.349321+0000 mgr.y (mgr.25723) 1168 : cluster 0 pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/58362 objects degraded (16.298%); 5970/58362 objects misplaced (10.229%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:32.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:32.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:32 smithi155 bash[59495]: cluster 2023-10-09T06:43:31.349321+0000 mgr.y (mgr.25723) 1168 : cluster 0 pgmap v1244: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 9512/58362 objects degraded (16.298%); 5970/58362 objects misplaced (10.229%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:43:32.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.926 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:32.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:33.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:33.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:33.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:33.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:33.206 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:33.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:33.328 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:33.437 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:33.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:33.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:33.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:33.728 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:33.832 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:33.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:33.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:33.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:34.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:34.127 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:34.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:34.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:34.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:34.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:34.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:34.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:34.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:34.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[80002]: cluster 2023-10-09T06:43:33.349943+0000 mgr.y (mgr.25723) 1169 : cluster 0 pgmap v1245: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 9512/56571 objects degraded (16.814%); 5776/56571 objects misplaced (10.210%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:43:34.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[80002]: audit 2023-10-09T06:43:33.603782+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:34.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[80002]: cluster 2023-10-09T06:43:34.060250+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 9512/56571 objects degraded (16.814%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:34.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[77299]: cluster 2023-10-09T06:43:33.349943+0000 mgr.y (mgr.25723) 1169 : cluster 0 pgmap v1245: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 9512/56571 objects degraded (16.814%); 5776/56571 objects misplaced (10.210%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:43:34.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[77299]: audit 2023-10-09T06:43:33.603782+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:34.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:34 smithi125 bash[77299]: cluster 2023-10-09T06:43:34.060250+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 9512/56571 objects degraded (16.814%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:34 smithi155 bash[59495]: cluster 2023-10-09T06:43:33.349943+0000 mgr.y (mgr.25723) 1169 : cluster 0 pgmap v1245: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 9512/56571 objects degraded (16.814%); 5776/56571 objects misplaced (10.210%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:43:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:34 smithi155 bash[59495]: audit 2023-10-09T06:43:33.603782+0000 mon.a (mon.0) 1444 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:34 smithi155 bash[59495]: cluster 2023-10-09T06:43:34.060250+0000 mon.a (mon.0) 1445 : cluster 3 Health check update: Degraded data redundancy: 9512/56571 objects degraded (16.814%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:34.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:34.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:34.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:34.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:34.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:34.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:34.999 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:43:35.000 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:43:35.000 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:43:35.000 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:43:35.000 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:43:35.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:35.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:35.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:35.195 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:35.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:35.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:35.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:35.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:35.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:35.602 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:35.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:35.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:35.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:35.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:35.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:35.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:36.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:36.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:36.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:36.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:43:36.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:36.257 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:36.287 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:36.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:36.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:36.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:36.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:36.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:36.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:36 smithi125 bash[77299]: cluster 2023-10-09T06:43:35.352367+0000 mgr.y (mgr.25723) 1170 : cluster 0 pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9512/49272 objects degraded (19.305%); 5072/49272 objects misplaced (10.294%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:36.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:36 smithi125 bash[80002]: cluster 2023-10-09T06:43:35.352367+0000 mgr.y (mgr.25723) 1170 : cluster 0 pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9512/49272 objects degraded (19.305%); 5072/49272 objects misplaced (10.294%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:36.694 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:36.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:36.844 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:36.869 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:36.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:36.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:36 smithi155 bash[59495]: cluster 2023-10-09T06:43:35.352367+0000 mgr.y (mgr.25723) 1170 : cluster 0 pgmap v1246: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 9512/49272 objects degraded (19.305%); 5072/49272 objects misplaced (10.294%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:43:36.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:36.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:36.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:37.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:37.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:37.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:37.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:37.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:37.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:37.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:37.529 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:37.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:37.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:37.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:37.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:37.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:37.908 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:37.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:37.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:38.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:38.071 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:38.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:38.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:38.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:38.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:38.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:38.473 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:38.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:38 smithi125 bash[77299]: cluster 2023-10-09T06:43:37.352918+0000 mgr.y (mgr.25723) 1171 : cluster 0 pgmap v1247: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1004 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/48039 objects degraded (19.801%); 4850/48039 objects misplaced (10.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:43:38.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:38 smithi125 bash[80002]: cluster 2023-10-09T06:43:37.352918+0000 mgr.y (mgr.25723) 1171 : cluster 0 pgmap v1247: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1004 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/48039 objects degraded (19.801%); 4850/48039 objects misplaced (10.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:43:38.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:38.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:38.706 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:38.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:38.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:38.847 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:38 smithi155 bash[59495]: cluster 2023-10-09T06:43:37.352918+0000 mgr.y (mgr.25723) 1171 : cluster 0 pgmap v1247: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1004 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 9512/48039 objects degraded (19.801%); 4850/48039 objects misplaced (10.096%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:43:38.865 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:38.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:38.973 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:38.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:39.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:39.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:39.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:39.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:39.264 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:39.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:39.517 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:39.549 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:39.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:39.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:39 smithi125 bash[77299]: cluster 2023-10-09T06:43:39.061317+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 9512/48039 objects degraded (19.801%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:39.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:39 smithi125 bash[80002]: cluster 2023-10-09T06:43:39.061317+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 9512/48039 objects degraded (19.801%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:39.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.801 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:39.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:39 smithi155 bash[59495]: cluster 2023-10-09T06:43:39.061317+0000 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 9512/48039 objects degraded (19.801%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:39.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:39.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:40.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:40.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:40.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:40.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:40.217 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:40.321 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:40.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:40.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:40.489 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:40.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:40.635 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:40 smithi125 bash[77299]: cluster 2023-10-09T06:43:39.353538+0000 mgr.y (mgr.25723) 1172 : cluster 0 pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 928 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9512/44406 objects degraded (21.421%); 4850/44406 objects misplaced (10.922%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:40.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:40 smithi125 bash[80002]: cluster 2023-10-09T06:43:39.353538+0000 mgr.y (mgr.25723) 1172 : cluster 0 pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 928 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9512/44406 objects degraded (21.421%); 4850/44406 objects misplaced (10.922%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:40.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:40.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:40.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:40.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:40.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:40 smithi155 bash[59495]: cluster 2023-10-09T06:43:39.353538+0000 mgr.y (mgr.25723) 1172 : cluster 0 pgmap v1248: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 928 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 9512/44406 objects degraded (21.421%); 4850/44406 objects misplaced (10.922%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:40.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:40.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:40.962 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:43:41.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.034 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:41.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:41.218 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:41] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.33.4" 2023-10-09T06:43:41.271 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:41.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:41.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:41.432 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:41.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:41.541 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.568 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:41.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:41.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:41.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:41.939 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:41.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:41.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:42.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:42.200 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:42.238 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:42.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:42.477 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:42.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:42.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:42.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:42 smithi155 bash[59495]: cluster 2023-10-09T06:43:41.354234+0000 mgr.y (mgr.25723) 1173 : cluster 0 pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 832 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 9512/39792 objects degraded (23.904%); 4064/39792 objects misplaced (10.213%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:42.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:42.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:42.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:42.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:42 smithi125 bash[77299]: cluster 2023-10-09T06:43:41.354234+0000 mgr.y (mgr.25723) 1173 : cluster 0 pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 832 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 9512/39792 objects degraded (23.904%); 4064/39792 objects misplaced (10.213%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:42.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:42 smithi125 bash[80002]: cluster 2023-10-09T06:43:41.354234+0000 mgr.y (mgr.25723) 1173 : cluster 0 pgmap v1249: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 832 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 9512/39792 objects degraded (23.904%); 4064/39792 objects misplaced (10.213%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:43:42.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:42.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:43.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:43.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:43.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:43.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:43.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:43.285 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:43.392 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:43.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:43.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:43.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:43.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:43.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:43.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:43.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:43.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:44.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:44.199 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:44.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:44.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:44.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:44.482 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:44.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:44.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:44 smithi125 bash[77299]: cluster 2023-10-09T06:43:43.354862+0000 mgr.y (mgr.25723) 1174 : cluster 0 pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 795 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9512/38031 objects degraded (25.011%); 3865/38031 objects misplaced (10.163%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:44.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:44 smithi125 bash[77299]: cluster 2023-10-09T06:43:44.062352+0000 mon.a (mon.0) 1447 : cluster 3 Health check update: Degraded data redundancy: 9512/38031 objects degraded (25.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:44 smithi125 bash[80002]: cluster 2023-10-09T06:43:43.354862+0000 mgr.y (mgr.25723) 1174 : cluster 0 pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 795 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9512/38031 objects degraded (25.011%); 3865/38031 objects misplaced (10.163%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:44 smithi125 bash[80002]: cluster 2023-10-09T06:43:44.062352+0000 mon.a (mon.0) 1447 : cluster 3 Health check update: Degraded data redundancy: 9512/38031 objects degraded (25.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:44.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:44 smithi155 bash[59495]: cluster 2023-10-09T06:43:43.354862+0000 mgr.y (mgr.25723) 1174 : cluster 0 pgmap v1250: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 795 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 9512/38031 objects degraded (25.011%); 3865/38031 objects misplaced (10.163%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:44 smithi155 bash[59495]: cluster 2023-10-09T06:43:44.062352+0000 mon.a (mon.0) 1447 : cluster 3 Health check update: Degraded data redundancy: 9512/38031 objects degraded (25.011%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:44.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:44.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:44.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:44.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:44.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:45.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:45.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:45.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:45.166 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:45.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:45.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:45.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:45.566 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:45.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.677 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:45 smithi155 bash[59495]: cluster 2023-10-09T06:43:45.355710+0000 mgr.y (mgr.25723) 1175 : cluster 0 pgmap v1251: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 627 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 9512/29970 objects degraded (31.738%); 3044/29970 objects misplaced (10.157%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:43:45.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:45.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:45.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:45 smithi125 bash[77299]: cluster 2023-10-09T06:43:45.355710+0000 mgr.y (mgr.25723) 1175 : cluster 0 pgmap v1251: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 627 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 9512/29970 objects degraded (31.738%); 3044/29970 objects misplaced (10.157%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:43:45.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:45 smithi125 bash[80002]: cluster 2023-10-09T06:43:45.355710+0000 mgr.y (mgr.25723) 1175 : cluster 0 pgmap v1251: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 627 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 9512/29970 objects degraded (31.738%); 3044/29970 objects misplaced (10.157%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:43:45.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:45.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:45.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:46.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:46.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:46.165 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:43:46.165 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:210147 8.10 8.6ddaf850 (undecoded) ondisk+write+known_if_redirected e210)", 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:51.709028+0000", 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.44540986700002, 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.66909228200000004, 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.166 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 210147 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709028+0000", 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709028+0000", 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709026+0000", 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.08e-06 2023-10-09T06:43:46.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709033+0000", 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2003e-05 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709045+0000", 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8240000000000002e-06 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709052+0000", 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0059999999999998e-05 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709112+0000", 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2549999999999997e-06 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.709120+0000", 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.66899118899999999 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378112+0000", 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7560000000000005e-06 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.212 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.212 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378120+0000", 2023-10-09T06:43:46.212 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.66909228200000004 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:210173 8.10 8.e1e1fc70 (undecoded) ondisk+write+known_if_redirected e210)", 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:51.741785+0000", 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.41265241600001, 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63637055300000001, 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.213 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 210173 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741785+0000", 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.214 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741785+0000", 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741783+0000", 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7670000000000003e-06 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.215 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741791+0000", 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4457999999999999e-05 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741805+0000", 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1550000000000001e-06 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.216 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741813+0000", 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7104e-05 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741880+0000", 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1939999999999999e-05 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.217 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.741892+0000", 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63625488500000005 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378147+0000", 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4520000000000003e-06 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378156+0000", 2023-10-09T06:43:46.218 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.63637055300000001 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:210227 8.10 8.1b8f57f0 (undecoded) ondisk+write+known_if_redirected e210)", 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:51.827186+0000", 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.32725198700001, 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.55100431599999999, 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:43:46.219 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 210227 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827186+0000", 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827186+0000", 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827184+0000", 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1391e-05 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.221 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827195+0000", 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1101999999999999e-05 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827216+0000", 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4320000000000005e-06 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827226+0000", 2023-10-09T06:43:46.222 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9039000000000001e-05 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827295+0000", 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3424999999999999e-05 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.827308+0000", 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.55087396 2023-10-09T06:43:46.223 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378182+0000", 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2560000000000002e-06 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378190+0000", 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.55100431599999999 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.224 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:210237 8.10 8.8ec908b0 (undecoded) ondisk+write+known_if_redirected e210)", 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:51.841497+0000", 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.31294056600001, 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.53672876700000005, 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.225 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 210237 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841497+0000", 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.226 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841497+0000", 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841496+0000", 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3980000000000002e-06 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841503+0000", 2023-10-09T06:43:46.227 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2992e-05 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841516+0000", 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1280000000000002e-06 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841523+0000", 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5500999999999997e-05 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.228 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841569+0000", 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7499999999999997e-06 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.841575+0000", 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.536642124 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.229 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378218+0000", 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.4509999999999992e-06 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378226+0000", 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.53672876700000005 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.230 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:210258 8.10 8.e3624990 (undecoded) ondisk+write+known_if_redirected e210)", 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:51.894575+0000", 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 354.25986284099997, 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.48368631099999998, 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 210258 2023-10-09T06:43:46.231 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894575+0000", 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894575+0000", 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.232 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894573+0000", 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1049999999999995e-06 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894582+0000", 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3804e-05 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.233 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894595+0000", 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3589999999999999e-06 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894601+0000", 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5219999999999997e-05 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.234 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894646+0000", 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5870000000000004e-06 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:51.894654+0000", 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.48359961699999998 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378253+0000", 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1759999999999993e-06 2023-10-09T06:43:46.235 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:52.378261+0000", 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.48368631099999998 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212463 8.10 8:0b77a811:::benchmark_data_smithi125_176369_object103395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.412355+0000", 2023-10-09T06:43:46.236 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.742082638, 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0650741290000001, 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212463 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.237 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412355+0000", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412355+0000", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412353+0000", 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6709999999999999e-06 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412361+0000", 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3997e-05 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412375+0000", 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.9299999999999992e-06 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.239 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.412384+0000", 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001897422 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.414281+0000", 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3040000000000001e-06 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T06:43:46.240 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.414288+0000", 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041682120000000001 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.418456+0000", 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2776999999999999e-05 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.418479+0000", 2023-10-09T06:43:46.241 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0265901559999999 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.445069+0000", 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000118121 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.445187+0000", 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034570400000000001 2023-10-09T06:43:46.242 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.445533+0000", 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0077192249999999997 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.453252+0000", 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024017999000000002 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.243 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477270+0000", 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8886000000000004e-05 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477339+0000", 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4156999999999997e-05 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.244 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477393+0000", 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6519000000000001e-05 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477429+0000", 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0650741290000001 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.245 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212525 8.10 8:0f152770:::benchmark_data_smithi125_176369_object103457:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.487837+0000", 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.66660069599999, 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98979539900000002, 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.246 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212525 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487837+0000", 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487837+0000", 2023-10-09T06:43:46.247 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999971 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487834+0000", 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3538e-05 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487848+0000", 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4170000000000001e-05 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.248 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487872+0000", 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0489999999999993e-06 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487881+0000", 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8857999999999997e-05 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.249 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487950+0000", 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2889e-05 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.487963+0000", 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95926729899999996 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.250 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.447230+0000", 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000142254 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.447372+0000", 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000352689 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.447725+0000", 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080729289999999995 2023-10-09T06:43:46.251 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.455798+0000", 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021696726 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477495+0000", 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4585e-05 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.252 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477549+0000", 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.672e-05 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477596+0000", 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6616999999999999e-05 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.253 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477633+0000", 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98979539900000002 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212534 8.10 8:0d3ac00e:::benchmark_data_smithi125_176369_object103466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.500245+0000", 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.654192927, 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.977630043, 2023-10-09T06:43:46.254 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212534 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500245+0000", 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.255 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500245+0000", 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500243+0000", 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1508999999999999e-05 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.256 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500254+0000", 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1508000000000001e-05 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500276+0000", 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3719e-05 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.257 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500290+0000", 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0677000000000002e-05 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500350+0000", 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0535e-05 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.500361+0000", 2023-10-09T06:43:46.258 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.94924613400000002 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.449607+0000", 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000158009 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.449765+0000", 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031672999999999998 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.259 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.450082+0000", 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057242839999999996 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.455806+0000", 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.021857822999999998 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.260 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477664+0000", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151108 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477815+0000", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4607000000000003e-05 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477849+0000", 2023-10-09T06:43:46.261 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5551999999999999e-05 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477875+0000", 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.977630043 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.262 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212579 8.10 8:0be2aa78:::benchmark_data_smithi125_176369_object103511:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.545329+0000", 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.60910936200003, 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93272943500000005, 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212579 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.263 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545329+0000", 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545329+0000", 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.264 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545327+0000", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6730000000000003e-06 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545333+0000", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5987000000000001e-05 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545349+0000", 2023-10-09T06:43:46.265 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7950000000000002e-06 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545356+0000", 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8343000000000001e-05 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545405+0000", 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0900000000000005e-06 2023-10-09T06:43:46.266 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.545413+0000", 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.906162676 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.451575+0000", 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8762000000000002e-05 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.267 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.451664+0000", 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024768499999999998 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.451912+0000", 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0079164920000000007 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.268 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.459828+0000", 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018119803 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477948+0000", 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9291e-05 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477997+0000", 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4517999999999997e-05 2023-10-09T06:43:46.269 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478032+0000", 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6067999999999998e-05 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478058+0000", 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.93272943500000005 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.270 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212597 8.10 8:0cf0004d:::benchmark_data_smithi125_176369_object103529:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.567772+0000", 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.586665676, 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91042187799999996, 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.271 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212597 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567772+0000", 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.272 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567772+0000", 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567770+0000", 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3729999999999993e-06 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567778+0000", 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.451e-05 2023-10-09T06:43:46.273 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567793+0000", 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2230000000000001e-06 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567800+0000", 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0970000000000001e-05 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.274 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567861+0000", 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1593000000000001e-05 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.567872+0000", 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.885410064 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.275 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.453282+0000", 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0001094 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.453392+0000", 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025697799999999999 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.453649+0000", 2023-10-09T06:43:46.276 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096128659999999994 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.463262+0000", 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014712776 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.477974+0000", 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016984699999999999 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.277 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478144+0000", 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7032000000000001e-05 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478171+0000", 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2728999999999999e-05 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.278 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478194+0000", 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.91042187799999996 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212611 8.10 8:087c878f:::benchmark_data_smithi125_176369_object103543:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.579557+0000", 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.57488052399998, 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89875015899999999, 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.279 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212611 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579557+0000", 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.280 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579557+0000", 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579555+0000", 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1889999999999996e-06 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.281 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579565+0000", 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5372000000000001e-05 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579580+0000", 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3839999999999997e-06 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579588+0000", 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1801e-05 2023-10-09T06:43:46.282 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579650+0000", 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1053000000000001e-05 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.579661+0000", 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87618013100000003 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.283 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.455841+0000", 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014402600000000001 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.455985+0000", 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00032930900000000001 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.284 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.456315+0000", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069547960000000001 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.463269+0000", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014840357 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478110+0000", 2023-10-09T06:43:46.285 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000122569 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478232+0000", 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2466999999999998e-05 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478275+0000", 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2727e-05 2023-10-09T06:43:46.286 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478308+0000", 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.89875015899999999 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212630 8.10 8:0b958873:::benchmark_data_smithi125_176369_object103562:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.605018+0000", 2023-10-09T06:43:46.287 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.54941989600002, 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87342529700000004, 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212630 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605018+0000", 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605018+0000", 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999976 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605016+0000", 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3281999999999999e-05 2023-10-09T06:43:46.289 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605029+0000", 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2507e-05 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605051+0000", 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0192e-05 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.290 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605062+0000", 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015278399999999999 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605214+0000", 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0597999999999999e-05 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.291 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.605225+0000", 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.85253560699999997 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.457761+0000", 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8245000000000006e-05 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.457859+0000", 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00025823600000000001 2023-10-09T06:43:46.292 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.458117+0000", 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087078910000000006 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.466825+0000", 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011521287999999999 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.293 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478346+0000", 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4921e-05 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478381+0000", 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4394999999999998e-05 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.294 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478416+0000", 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.7684000000000001e-05 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478443+0000", 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.87342529700000004 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.295 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212655 8.10 8:083d8abf:::benchmark_data_smithi125_176369_object103587:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.634055+0000", 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.52038290600001, 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84455196600000004, 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212655 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.296 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634055+0000", 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634055+0000", 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.297 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634053+0000", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2819999999999999e-06 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634060+0000", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.27e-05 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634073+0000", 2023-10-09T06:43:46.298 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1289999999999998e-06 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634080+0000", 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8288999999999997e-05 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634139+0000", 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9009999999999997e-06 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.299 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.634145+0000", 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.82571643800000005 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.459862+0000", 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2058999999999995e-05 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.300 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.459944+0000", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022268900000000001 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.460167+0000", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066620409999999996 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.466829+0000", 2023-10-09T06:43:46.301 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011652782 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478481+0000", 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6433000000000005e-05 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478558+0000", 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.6265e-05 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.302 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478584+0000", 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2793999999999999e-05 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478607+0000", 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.84455196600000004 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.303 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212661 8.10 8:0a6c7e02:::benchmark_data_smithi125_176369_object103593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.641537+0000", 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.512900683, 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83720895299999998, 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212661 2023-10-09T06:43:46.304 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641537+0000", 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641537+0000", 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.305 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641535+0000", 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6079999999999998e-06 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641543+0000", 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5758000000000001e-05 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.306 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641559+0000", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6810000000000001e-06 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.641565+0000", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0016982659999999999 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.643264+0000", 2023-10-09T06:43:46.307 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8799999999999997e-06 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.643272+0000", 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.81809954799999995 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.461372+0000", 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012469 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.308 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.461497+0000", 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024498899999999998 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.461742+0000", 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0087736989999999994 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.309 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.470515+0000", 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0080170889999999998 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478532+0000", 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000111263 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.310 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478644+0000", 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9230999999999997e-05 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478693+0000", 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3189999999999997e-05 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.311 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478746+0000", 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.83720895299999998 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212689 8.10 8:0cb52e07:::benchmark_data_smithi125_176369_object103621:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.679249+0000", 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.47518916000001, 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79969514600000002, 2023-10-09T06:43:46.312 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212689 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679249+0000", 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.313 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679249+0000", 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679247+0000", 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0580000000000003e-06 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.314 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679253+0000", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2432999999999999e-05 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679266+0000", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2939999999999996e-06 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679271+0000", 2023-10-09T06:43:46.315 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8174000000000003e-05 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679319+0000", 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7710000000000001e-06 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.679326+0000", 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.78400546900000001 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.316 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.463331+0000", 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000141352 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.463473+0000", 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00028052200000000001 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.317 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.463753+0000", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0067681319999999996 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.470521+0000", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0082892899999999995 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478811+0000", 2023-10-09T06:43:46.318 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9752000000000002e-05 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478850+0000", 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7151000000000003e-05 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478898+0000", 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6267000000000003e-05 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.319 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478944+0000", 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.79969514600000002 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212702 8.10 8:0cdfb2ae:::benchmark_data_smithi125_176369_object103634:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.700052+0000", 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.45438580500002, 2023-10-09T06:43:46.320 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77909961400000005, 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212702 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700052+0000", 2023-10-09T06:43:46.321 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700052+0000", 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700050+0000", 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2269999999999996e-06 2023-10-09T06:43:46.322 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700058+0000", 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3225e-05 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700071+0000", 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.9599999999999997e-06 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.323 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700077+0000", 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8835999999999997e-05 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700136+0000", 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.644e-06 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.324 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.700143+0000", 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.76515265499999996 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.465296+0000", 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2365000000000001e-05 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.465388+0000", 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00021943 2023-10-09T06:43:46.325 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.465608+0000", 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091023939999999998 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.474710+0000", 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0042880169999999999 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.326 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.478998+0000", 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7173999999999999e-05 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479045+0000", 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8950999999999999e-05 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.327 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479104+0000", 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7326000000000001e-05 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479152+0000", 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.77909961400000005 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.328 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212740 8.10 8:0f57e140:::benchmark_data_smithi125_176369_object103672:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.755940+0000", 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.39849828600001, 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.72343136500000005, 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212740 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.329 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755940+0000", 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755940+0000", 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.330 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755938+0000", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2499999999999995e-06 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755947+0000", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8459999999999999e-05 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755965+0000", 2023-10-09T06:43:46.331 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0050000000000004e-06 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.755973+0000", 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0878000000000002e-05 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.756034+0000", 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0937e-05 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.332 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.756045+0000", 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.71083351900000002 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.466879+0000", 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000140099 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.333 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.467019+0000", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00028388699999999997 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.467303+0000", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074164219999999998 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.474719+0000", 2023-10-09T06:43:46.334 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045150370000000004 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479234+0000", 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4151999999999999e-05 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479288+0000", 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5264e-05 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.335 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479324+0000", 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7363e-05 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479371+0000", 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.72343136500000005 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.336 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212753 8.10 8:0e75cdd5:::benchmark_data_smithi125_176369_object103685:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.780332+0000", 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.37410582899997, 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.69929182199999995, 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.337 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212753 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780332+0000", 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.338 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780332+0000", 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780330+0000", 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1110000000000008e-06 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.339 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780338+0000", 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2497000000000001e-05 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780351+0000", 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3799999999999999e-06 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780357+0000", 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6017000000000002e-05 2023-10-09T06:43:46.340 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780423+0000", 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4243000000000001e-05 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.780437+0000", 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.68852247899999997 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.341 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.468960+0000", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114059 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.469074+0000", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000217627 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.469291+0000", 2023-10-09T06:43:46.342 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074962730000000003 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.476788+0000", 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026426869999999999 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479430+0000", 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9225000000000002e-05 2023-10-09T06:43:46.343 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479500+0000", 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1594999999999996e-05 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479561+0000", 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2596999999999997e-05 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.344 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479624+0000", 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.69929182199999995 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45383.0:212828 8.10 8:09fcdefd:::benchmark_data_smithi125_176369_object103760:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:37:56.919718+0000", 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 349.234719638, 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.56008438999999999, 2023-10-09T06:43:46.345 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45383", 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 212828 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919718+0000", 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.346 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919718+0000", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919716+0000", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.7029999999999995e-06 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919725+0000", 2023-10-09T06:43:46.347 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4919000000000001e-05 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919740+0000", 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4359999999999996e-06 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.349 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919747+0000", 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3106999999999999e-05 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919791+0000", 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7479999999999996e-06 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.350 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:56.919797+0000", 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.55078474499999996 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.470582+0000", 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017625099999999999 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.470758+0000", 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000333791 2023-10-09T06:43:46.351 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.471092+0000", 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057089740000000003 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.476801+0000", 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028697549999999999 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.352 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479671+0000", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2407000000000002e-05 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479713+0000", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9638000000000001e-05 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479763+0000", 2023-10-09T06:43:46.353 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9709999999999998e-05 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:37:57.479803+0000", 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.56008438999999999 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.354 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:155849 9.d 9:bf356b8d:::benchmark_data_smithi125_199533_object51318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 74.644740343999999, 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0983538209999999, 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 155849 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:43:46.355 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:43:46.356 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509694+0000", 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6396999999999999e-05 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509710+0000", 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6316999999999998e-05 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509747+0000", 2023-10-09T06:43:46.357 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7050000000000001e-05 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509764+0000", 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3009000000000005e-05 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509837+0000", 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5705e-05 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.358 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509853+0000", 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0938460160000001 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603699+0000", 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013094199999999999 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:43:46.359 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603830+0000", 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010252299999999999 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603932+0000", 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002793626 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.606726+0000", 2023-10-09T06:43:46.360 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042365099999999999 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607149+0000", 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042726500000000002 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607577+0000", 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035003900000000001 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.361 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607927+0000", 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9614000000000003e-05 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607976+0000", 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5104000000000005e-05 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:43:46.362 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.608051+0000", 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0983538209999999 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:43:46.363 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:43:46.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:46.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:46.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:46.518 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:46.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:46.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:46.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:46.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:46.800 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:46.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:46.936 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:46.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:47.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:47.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:47.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:47.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.209 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:47.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:43:47.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:47.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:47.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:47.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:47.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:47.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:47.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:47.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:47.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:47.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:47.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:48.011 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:48.045 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:43:48.070 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:43:48.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:48.115 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[77299]: debug 2023-10-09T06:43:48.099+0000 7ff949fd9700 -1 mon.a@0(leader).osd e269 definitely_dead 0 2023-10-09T06:43:48.137 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:43:48.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:48.194 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:43:48.239 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:43:48.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:48.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:48.273 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:43:48.301 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:43:48.336 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:43:48.347 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.347 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.348 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:43:48.372 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:48.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:48.431 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.1. 2023-10-09T06:43:48.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:48.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:48.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:48.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[77299]: cluster 2023-10-09T06:43:47.356611+0000 mgr.y (mgr.25723) 1176 : cluster 0 pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 601 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 9512/28710 objects degraded (33.131%); 2830/28710 objects misplaced (9.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:48.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[77299]: audit 2023-10-09T06:43:47.990521+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.125:0/3874107775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:48.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[77299]: audit 2023-10-09T06:43:48.106315+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T06:43:48.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[77299]: cluster 2023-10-09T06:43:48.106432+0000 mon.a (mon.0) 1450 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T06:43:48.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[80002]: cluster 2023-10-09T06:43:47.356611+0000 mgr.y (mgr.25723) 1176 : cluster 0 pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 601 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 9512/28710 objects degraded (33.131%); 2830/28710 objects misplaced (9.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:48.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[80002]: audit 2023-10-09T06:43:47.990521+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.125:0/3874107775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:48.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[80002]: audit 2023-10-09T06:43:48.106315+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T06:43:48.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:48 smithi125 bash[80002]: cluster 2023-10-09T06:43:48.106432+0000 mon.a (mon.0) 1450 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T06:43:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:48 smithi155 bash[59495]: cluster 2023-10-09T06:43:47.356611+0000 mgr.y (mgr.25723) 1176 : cluster 0 pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 80 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 601 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 9512/28710 objects degraded (33.131%); 2830/28710 objects misplaced (9.857%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:48 smithi155 bash[59495]: audit 2023-10-09T06:43:47.990521+0000 mon.a (mon.0) 1448 : audit 0 from='client.? 172.21.15.125:0/3874107775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:48 smithi155 bash[59495]: audit 2023-10-09T06:43:48.106315+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T06:43:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:48 smithi155 bash[59495]: cluster 2023-10-09T06:43:48.106432+0000 mon.a (mon.0) 1450 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T06:43:48.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:48.763 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:48.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:48.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:48.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:48.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:48.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:49.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:49.084 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:43:49.152 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:49.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:49.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:49.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:49.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:49.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:49.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.553 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:49.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:49.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:49.668 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: audit 2023-10-09T06:43:48.415051+0000 mon.a (mon.0) 1451 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:48.415127+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e270: 8 total, 8 up, 6 in 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:48.415269+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:48.419004+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: audit 2023-10-09T06:43:48.604294+0000 mon.a (mon.0) 1453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: audit 2023-10-09T06:43:48.707808+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: audit 2023-10-09T06:43:48.709934+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.125:0/2838589894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.063390+0000 mon.a (mon.0) 1455 : cluster 3 Health check update: Degraded data redundancy: 9512/28710 objects degraded (33.131%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.063805+0000 mon.a (mon.0) 1456 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: audit 2023-10-09T06:43:49.068795+0000 mon.a (mon.0) 1457 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:49.669 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:49 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.068898+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e271: 8 total, 8 up, 6 in 2023-10-09T06:43:49.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:49.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: audit 2023-10-09T06:43:48.415051+0000 mon.a (mon.0) 1451 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T06:43:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:48.415127+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e270: 8 total, 8 up, 6 in 2023-10-09T06:43:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:48.415269+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:48.419004+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: audit 2023-10-09T06:43:48.604294+0000 mon.a (mon.0) 1453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: audit 2023-10-09T06:43:48.707808+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: audit 2023-10-09T06:43:48.709934+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.125:0/2838589894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.063390+0000 mon.a (mon.0) 1455 : cluster 3 Health check update: Degraded data redundancy: 9512/28710 objects degraded (33.131%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.063805+0000 mon.a (mon.0) 1456 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: audit 2023-10-09T06:43:49.068795+0000 mon.a (mon.0) 1457 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.068898+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e271: 8 total, 8 up, 6 in 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: audit 2023-10-09T06:43:48.415051+0000 mon.a (mon.0) 1451 : audit 1 from='client.? 172.21.15.125:0/2535680374' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T06:43:49.798 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:48.415127+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e270: 8 total, 8 up, 6 in 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:48.415269+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:48.419004+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e270 with expected crc 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: audit 2023-10-09T06:43:48.604294+0000 mon.a (mon.0) 1453 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: audit 2023-10-09T06:43:48.707808+0000 mon.a (mon.0) 1454 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: audit 2023-10-09T06:43:48.709934+0000 mon.b (mon.1) 34 : audit 1 from='client.? 172.21.15.125:0/2838589894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.063390+0000 mon.a (mon.0) 1455 : cluster 3 Health check update: Degraded data redundancy: 9512/28710 objects degraded (33.131%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.063805+0000 mon.a (mon.0) 1456 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: audit 2023-10-09T06:43:49.068795+0000 mon.a (mon.0) 1457 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:49.799 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:49 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.068898+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e271: 8 total, 8 up, 6 in 2023-10-09T06:43:49.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:49.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:49.938 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:49.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:50.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:50.068 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:50.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:50.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:50.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:50.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:50.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:50.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:50.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:50.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:50.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:50.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:50.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:50.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:50.730 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.071689+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.730 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.073181+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.073211+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.076410+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.078826+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.078974+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.079008+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.080475+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.080742+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.082607+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.248142+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:49.357032+0000 mgr.y (mgr.25723) 1177 : cluster 0 pgmap v1255: 89 pgs: 2 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 526 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 9512/25101 objects degraded (37.895%); 2830/25101 objects misplaced (11.274%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:50.731 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:50 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.084128+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e272: 8 total, 8 up, 6 in 2023-10-09T06:43:50.744 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:50.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:50.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.071689+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.073181+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.073211+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.076410+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.078826+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.078974+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.079008+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.080475+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.080742+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.082607+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.248142+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:49.357032+0000 mgr.y (mgr.25723) 1177 : cluster 0 pgmap v1255: 89 pgs: 2 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 526 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 9512/25101 objects degraded (37.895%); 2830/25101 objects misplaced (11.274%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:50.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.084128+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e272: 8 total, 8 up, 6 in 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.071689+0000 osd.4 (osd.4) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.073181+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.073211+0000 osd.7 (osd.7) 205 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.076410+0000 osd.4 (osd.4) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.078826+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.078974+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.079008+0000 osd.4 (osd.4) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.080475+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.080742+0000 osd.4 (osd.4) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.082607+0000 osd.4 (osd.4) 224 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.861 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.248142+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e271 with expected crc 2023-10-09T06:43:50.861 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:49.357032+0000 mgr.y (mgr.25723) 1177 : cluster 0 pgmap v1255: 89 pgs: 2 peering, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 526 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 611 op/s; 9512/25101 objects degraded (37.895%); 2830/25101 objects misplaced (11.274%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:43:50.861 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:50 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.084128+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e272: 8 total, 8 up, 6 in 2023-10-09T06:43:50.889 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:50.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:50.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:51.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:51.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:51.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:51.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:51.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.086557+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.087268+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.087374+0000 osd.3 (osd.3) 16 : cluster 0 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 269'12606 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.089096+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.089569+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.091567+0000 osd.4 (osd.4) 227 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 269'12793 2023-10-09T06:43:51.559 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.094630+0000 osd.7 (osd.7) 209 : cluster 0 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 269'12513 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[77299]: cluster 2023-10-09T06:43:50.108585+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:43:51] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.086557+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.087268+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.087374+0000 osd.3 (osd.3) 16 : cluster 0 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 269'12606 2023-10-09T06:43:51.560 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.089096+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.089569+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.091567+0000 osd.4 (osd.4) 227 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 269'12793 2023-10-09T06:43:51.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.094630+0000 osd.7 (osd.7) 209 : cluster 0 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 269'12513 2023-10-09T06:43:51.561 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:51 smithi125 bash[80002]: cluster 2023-10-09T06:43:50.108585+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:51.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.086557+0000 osd.4 (osd.4) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.087268+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.087374+0000 osd.3 (osd.3) 16 : cluster 0 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 269'12606 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.089096+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.089569+0000 osd.4 (osd.4) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.091567+0000 osd.4 (osd.4) 227 : cluster 0 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 269'12793 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.094630+0000 osd.7 (osd.7) 209 : cluster 0 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 269'12513 2023-10-09T06:43:51.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:51 smithi155 bash[59495]: cluster 2023-10-09T06:43:50.108585+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e272 with expected crc 2023-10-09T06:43:51.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:51.840 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:51.889 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:51.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:51.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:51.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:52.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:52.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:52.258 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:52.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:52.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:52.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:52.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:52.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:52.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:52 smithi155 bash[59495]: cluster 2023-10-09T06:43:51.357568+0000 mgr.y (mgr.25723) 1178 : cluster 0 pgmap v1257: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 448 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 9512/21375 objects degraded (44.501%); 2216/21375 objects misplaced (10.367%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:43:52.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:52.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:52.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:52.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:52.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:52 smithi125 bash[77299]: cluster 2023-10-09T06:43:51.357568+0000 mgr.y (mgr.25723) 1178 : cluster 0 pgmap v1257: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 448 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 9512/21375 objects degraded (44.501%); 2216/21375 objects misplaced (10.367%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:43:52.891 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:52 smithi125 bash[80002]: cluster 2023-10-09T06:43:51.357568+0000 mgr.y (mgr.25723) 1178 : cluster 0 pgmap v1257: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 1 active+recovering+undersized+remapped, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 448 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 9512/21375 objects degraded (44.501%); 2216/21375 objects misplaced (10.367%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:43:52.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:52.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:52.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:53.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:53.092 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:53.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:53.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:53.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:53.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:53.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:53.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:53.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:53.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:53.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:53.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:53.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:53.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:54.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:54.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:54.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:54.167 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:54.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:54.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:54.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:54.427 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:54.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:54.542 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.358989+0000 mgr.y (mgr.25723) 1179 : cluster 0 pgmap v1258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 426 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 11911/20301 objects degraded (58.672%); 1745/20301 objects misplaced (8.596%); 1.3 MiB/s, 77 objects/s recovering 2023-10-09T06:43:54.542 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.449915+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e273: 8 total, 8 up, 6 in 2023-10-09T06:43:54.542 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.457584+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.542 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.461175+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.463715+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:53.948660+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.543 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:54 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.064608+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 11911/20301 objects degraded (58.672%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:54.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:54.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:54.706 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.358989+0000 mgr.y (mgr.25723) 1179 : cluster 0 pgmap v1258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 426 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 11911/20301 objects degraded (58.672%); 1745/20301 objects misplaced (8.596%); 1.3 MiB/s, 77 objects/s recovering 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.449915+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e273: 8 total, 8 up, 6 in 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.457584+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.461175+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.463715+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:53.948660+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.064608+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 11911/20301 objects degraded (58.672%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.358989+0000 mgr.y (mgr.25723) 1179 : cluster 0 pgmap v1258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 remapped+peering, 2 peering, 76 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 426 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 466 op/s; 11911/20301 objects degraded (58.672%); 1745/20301 objects misplaced (8.596%); 1.3 MiB/s, 77 objects/s recovering 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.449915+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e273: 8 total, 8 up, 6 in 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.457584+0000 osd.4 (osd.4) 228 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.461175+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.463715+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:53.948660+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e273 with expected crc 2023-10-09T06:43:54.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:54 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.064608+0000 mon.a (mon.0) 1461 : cluster 3 Health check update: Degraded data redundancy: 11911/20301 objects degraded (58.672%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:54.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:54.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:54.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:54.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:54.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:55.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:55.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:55.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:55.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:43:55.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:43:55.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:55.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:55.397 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:55.501 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.451814+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.453881+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e274: 8 total, 8 up, 6 in 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.457602+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.463240+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.464407+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:55 smithi155 bash[59495]: cluster 2023-10-09T06:43:54.629598+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:55.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:55.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:55.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.451814+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.453881+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e274: 8 total, 8 up, 6 in 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.457602+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.463240+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.464407+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[77299]: cluster 2023-10-09T06:43:54.629598+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.451814+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.453881+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e274: 8 total, 8 up, 6 in 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.457602+0000 osd.4 (osd.4) 229 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.463240+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.464407+0000 osd.4 (osd.4) 230 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:55 smithi125 bash[80002]: cluster 2023-10-09T06:43:54.629598+0000 osd.5 (osd.5) 211 : cluster 3 failed to encode map e274 with expected crc 2023-10-09T06:43:55.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:55.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:55.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:55.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:56.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:56.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:56.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:56.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:56.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:56.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:56.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:56.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:56.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:56.617 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:56 smithi125 bash[77299]: cluster 2023-10-09T06:43:55.361451+0000 mgr.y (mgr.25723) 1180 : cluster 0 pgmap v1261: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 314 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 11911/14925 objects degraded (79.806%); 1704/14925 objects misplaced (11.417%); 6.3 MiB/s, 156 objects/s recovering 2023-10-09T06:43:56.618 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:56 smithi125 bash[80002]: cluster 2023-10-09T06:43:55.361451+0000 mgr.y (mgr.25723) 1180 : cluster 0 pgmap v1261: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 314 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 11911/14925 objects degraded (79.806%); 1704/14925 objects misplaced (11.417%); 6.3 MiB/s, 156 objects/s recovering 2023-10-09T06:43:56.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:56.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:56.760 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:56 smithi155 bash[59495]: cluster 2023-10-09T06:43:55.361451+0000 mgr.y (mgr.25723) 1180 : cluster 0 pgmap v1261: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 314 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 11911/14925 objects degraded (79.806%); 1704/14925 objects misplaced (11.417%); 6.3 MiB/s, 156 objects/s recovering 2023-10-09T06:43:56.766 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:56.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:56.878 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:56.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:56.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:56.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:57.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:57.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:57.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:57.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:57.277 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:57.304 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:57.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:57.433 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:57.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:57.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:57.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:57.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:57.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:57.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:58.091 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:58.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:58.226 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 7, 6] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:43:58.226 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:43:58.226 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:43:58.226 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-10-09T06:43:58.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-10-09T06:43:58.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:58.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:58.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:58.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:58.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:58.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:58 smithi125 bash[77299]: cluster 2023-10-09T06:43:57.362040+0000 mgr.y (mgr.25723) 1181 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 293 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 11751/13914 objects degraded (84.455%); 1704/13914 objects misplaced (12.247%); 6.5 MiB/s, 149 objects/s recovering 2023-10-09T06:43:58.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:58 smithi125 bash[80002]: cluster 2023-10-09T06:43:57.362040+0000 mgr.y (mgr.25723) 1181 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 293 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 11751/13914 objects degraded (84.455%); 1704/13914 objects misplaced (12.247%); 6.5 MiB/s, 149 objects/s recovering 2023-10-09T06:43:58.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:58.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:58.841 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.850 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:58 smithi155 bash[59495]: cluster 2023-10-09T06:43:57.362040+0000 mgr.y (mgr.25723) 1181 : cluster 0 pgmap v1262: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+undersized+remapped, 77 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 293 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 11751/13914 objects degraded (84.455%); 1704/13914 objects misplaced (12.247%); 6.5 MiB/s, 149 objects/s recovering 2023-10-09T06:43:58.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:58.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.991 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:58.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:58.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:59.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:59.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:59.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:59.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:59.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:59.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:59.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:59.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:59.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:59.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:43:59 smithi125 bash[77299]: cluster 2023-10-09T06:43:59.066079+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 11751/13914 objects degraded (84.455%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:59.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:43:59 smithi125 bash[80002]: cluster 2023-10-09T06:43:59.066079+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 11751/13914 objects degraded (84.455%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.659 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:59.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:59.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:59.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:43:59 smithi155 bash[59495]: cluster 2023-10-09T06:43:59.066079+0000 mon.a (mon.0) 1463 : cluster 3 Health check update: Degraded data redundancy: 11751/13914 objects degraded (84.455%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:43:59.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:59.824 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:59.824 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:59.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:59.933 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:59.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:00.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:00.070 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:00.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:00.201 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:00.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:00.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:00.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:00.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:00.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:00 smithi125 bash[77299]: cluster 2023-10-09T06:43:59.362693+0000 mgr.y (mgr.25723) 1182 : cluster 0 pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 230 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 11751/10905 objects degraded (107.758%); 1704/10905 objects misplaced (15.626%); 5.9 MiB/s, 185 objects/s recovering 2023-10-09T06:44:00.593 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:00 smithi125 bash[80002]: cluster 2023-10-09T06:43:59.362693+0000 mgr.y (mgr.25723) 1182 : cluster 0 pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 230 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 11751/10905 objects degraded (107.758%); 1704/10905 objects misplaced (15.626%); 5.9 MiB/s, 185 objects/s recovering 2023-10-09T06:44:00.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:00.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:00.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:00.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:00 smithi155 bash[59495]: cluster 2023-10-09T06:43:59.362693+0000 mgr.y (mgr.25723) 1182 : cluster 0 pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 230 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 11751/10905 objects degraded (107.758%); 1704/10905 objects misplaced (15.626%); 5.9 MiB/s, 185 objects/s recovering 2023-10-09T06:44:00.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:00.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:00.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:01.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:01.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:01.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.317 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:01.347 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:01] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-10-09T06:44:01.420 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:01.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:01.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:01.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:01.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:01.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:01.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.852 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:01.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:01.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:01.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:01.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:02.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.109 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:02.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:02.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:02.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:02.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:02.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:02.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:02.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:02.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.363374+0000 mgr.y (mgr.25723) 1183 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 132 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 11751/6198 objects degraded (189.593%); 526/6198 objects misplaced (8.487%); 5.5 MiB/s, 137 objects/s recovering 2023-10-09T06:44:02.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.507307+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.508386+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e275: 8 total, 8 up, 6 in 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.508635+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.511309+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[77299]: cluster 2023-10-09T06:44:01.512719+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.363374+0000 mgr.y (mgr.25723) 1183 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 132 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 11751/6198 objects degraded (189.593%); 526/6198 objects misplaced (8.487%); 5.5 MiB/s, 137 objects/s recovering 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.507307+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.508386+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e275: 8 total, 8 up, 6 in 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.508635+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.511309+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.640 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:02 smithi125 bash[80002]: cluster 2023-10-09T06:44:01.512719+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:02.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:02.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.363374+0000 mgr.y (mgr.25723) 1183 : cluster 0 pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 132 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 11751/6198 objects degraded (189.593%); 526/6198 objects misplaced (8.487%); 5.5 MiB/s, 137 objects/s recovering 2023-10-09T06:44:02.751 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.507307+0000 osd.5 (osd.5) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.751 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.508386+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e275: 8 total, 8 up, 6 in 2023-10-09T06:44:02.751 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.508635+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.751 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.511309+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.751 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:02 smithi155 bash[59495]: cluster 2023-10-09T06:44:01.512719+0000 osd.4 (osd.4) 231 : cluster 3 failed to encode map e275 with expected crc 2023-10-09T06:44:02.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:02.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.882 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:02.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:02.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:03.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:03.097 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 104528 objects 2023-10-09T06:44:03.097 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :180.134 2023-10-09T06:44:03.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:03.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:03.424 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:03.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:03.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:03.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.506962+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e276: 8 total, 8 up, 6 in 2023-10-09T06:44:03.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.506981+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.509755+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.510059+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.512236+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.565 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.513107+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.565 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:02.514274+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.565 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:03 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.363936+0000 mgr.y (mgr.25723) 1184 : cluster 0 pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 98 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 11575/4584 objects degraded (252.509%); 526/4584 objects misplaced (11.475%); 3.2 MiB/s, 161 objects/s recovering 2023-10-09T06:44:03.578 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:03.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:03.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.506962+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e276: 8 total, 8 up, 6 in 2023-10-09T06:44:03.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.506981+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.509755+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.510059+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.512236+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.513107+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:02.514274+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.363936+0000 mgr.y (mgr.25723) 1184 : cluster 0 pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 98 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 11575/4584 objects degraded (252.509%); 526/4584 objects misplaced (11.475%); 3.2 MiB/s, 161 objects/s recovering 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.506962+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e276: 8 total, 8 up, 6 in 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.506981+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.509755+0000 osd.4 (osd.4) 232 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.510059+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.695 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.512236+0000 osd.4 (osd.4) 233 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.513107+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:02.514274+0000 osd.4 (osd.4) 234 : cluster 3 failed to encode map e276 with expected crc 2023-10-09T06:44:03.696 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:03 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.363936+0000 mgr.y (mgr.25723) 1184 : cluster 0 pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 78 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 98 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 430 op/s; 11575/4584 objects degraded (252.509%); 526/4584 objects misplaced (11.475%); 3.2 MiB/s, 161 objects/s recovering 2023-10-09T06:44:03.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:03.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:03.945 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:03.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:03.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:04.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:04.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:04.200 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:04.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:04.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:04.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:04.461 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:04.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:04.597 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:04.621 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.516976+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.621 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.519584+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e277: 8 total, 8 up, 6 in 2023-10-09T06:44:04.622 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.522726+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:04.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:04.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:04.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:04.857 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:04.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:04.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.525954+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: audit 2023-10-09T06:44:03.608393+0000 mon.a (mon.0) 1467 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: audit 2023-10-09T06:44:03.609949+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:03.679700+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.067202+0000 mon.a (mon.0) 1469 : cluster 3 Health check update: Degraded data redundancy: 11575/4584 objects degraded (252.509%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.516976+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.519584+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e277: 8 total, 8 up, 6 in 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.522726+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.525954+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: audit 2023-10-09T06:44:03.608393+0000 mon.a (mon.0) 1467 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:04.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: audit 2023-10-09T06:44:03.609949+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:04.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:03.679700+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.885 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:04 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.067202+0000 mon.a (mon.0) 1469 : cluster 3 Health check update: Degraded data redundancy: 11575/4584 objects degraded (252.509%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:04.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.516976+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.519584+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e277: 8 total, 8 up, 6 in 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.522726+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.525954+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: audit 2023-10-09T06:44:03.608393+0000 mon.a (mon.0) 1467 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: audit 2023-10-09T06:44:03.609949+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:03.679700+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e277 with expected crc 2023-10-09T06:44:04.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:04 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.067202+0000 mon.a (mon.0) 1469 : cluster 3 Health check update: Degraded data redundancy: 11575/4584 objects degraded (252.509%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:04.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:05.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:05.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:05.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:05.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:05.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:05.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:05.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:05.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:05.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:05.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.520064+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e278: 8 total, 8 up, 6 in 2023-10-09T06:44:05.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.521544+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.523974+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.526009+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:04.527625+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:05 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.364638+0000 mgr.y (mgr.25723) 1185 : cluster 0 pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 7.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 11575/228 objects degraded (5076.754%); 3.8 MiB/s, 183 objects/s recovering 2023-10-09T06:44:05.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:05.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:05.793 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.520064+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e278: 8 total, 8 up, 6 in 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.521544+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.523974+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.526009+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:04.527625+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.364638+0000 mgr.y (mgr.25723) 1185 : cluster 0 pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 7.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 11575/228 objects degraded (5076.754%); 3.8 MiB/s, 183 objects/s recovering 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.520064+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e278: 8 total, 8 up, 6 in 2023-10-09T06:44:05.794 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.521544+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.523974+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.526009+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:04.527625+0000 osd.4 (osd.4) 235 : cluster 3 failed to encode map e278 with expected crc 2023-10-09T06:44:05.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:05 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.364638+0000 mgr.y (mgr.25723) 1185 : cluster 0 pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 7.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 11575/228 objects degraded (5076.754%); 3.8 MiB/s, 183 objects/s recovering 2023-10-09T06:44:05.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:05.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:05.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:05.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:06.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:06.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:06.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:06.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.193 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:06.296 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.304 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T06:44:06.304 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T06:44:06.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:06.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:06.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:06.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:06.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:06.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:06 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.538792+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:06 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.538855+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:06 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.538991+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e279: 8 total, 8 up, 6 in 2023-10-09T06:44:06.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:06 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.542169+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:06 smithi155 bash[59495]: cluster 2023-10-09T06:44:05.544550+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:06.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:06.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:06.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.538792+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.538855+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.538991+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e279: 8 total, 8 up, 6 in 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.542169+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[77299]: cluster 2023-10-09T06:44:05.544550+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.538792+0000 osd.4 (osd.4) 236 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.538855+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.538991+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e279: 8 total, 8 up, 6 in 2023-10-09T06:44:06.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.542169+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.894 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:06 smithi125 bash[80002]: cluster 2023-10-09T06:44:05.544550+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e279 with expected crc 2023-10-09T06:44:06.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:06.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:06.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:07.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:07.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:07.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:07.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:07.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:07.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:07.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:07.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:07.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:07.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:07.609 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:06.542704+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e280: 8 total, 8 up, 6 in 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:06.542979+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:06.545906+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:06.546051+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:06.547315+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:07 smithi155 bash[59495]: cluster 2023-10-09T06:44:07.365440+0000 mgr.y (mgr.25723) 1186 : cluster 0 pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 5.3 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 7425/117 objects degraded (6346.154%); 1.3 MiB/s, 51 objects/s recovering 2023-10-09T06:44:07.732 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.751 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:07.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:07.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:07.927 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:06.542704+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e280: 8 total, 8 up, 6 in 2023-10-09T06:44:07.927 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:06.542979+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:06.545906+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:06.546051+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:06.547315+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[77299]: cluster 2023-10-09T06:44:07.365440+0000 mgr.y (mgr.25723) 1186 : cluster 0 pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 5.3 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 7425/117 objects degraded (6346.154%); 1.3 MiB/s, 51 objects/s recovering 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:06.542704+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e280: 8 total, 8 up, 6 in 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:06.542979+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:06.545906+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.928 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:06.546051+0000 osd.4 (osd.4) 237 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.929 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:06.547315+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e280 with expected crc 2023-10-09T06:44:07.929 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:07 smithi125 bash[80002]: cluster 2023-10-09T06:44:07.365440+0000 mgr.y (mgr.25723) 1186 : cluster 0 pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 5.3 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 372 op/s; 7425/117 objects degraded (6346.154%); 1.3 MiB/s, 51 objects/s recovering 2023-10-09T06:44:07.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:08.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:08.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:08.132 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:08.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:08.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:08.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:08.387 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:08.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:08.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:08.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:08.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:08.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:08.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:08.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:08.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:08.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:08.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:09.007 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:09.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:09.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:09.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:09.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:09.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:09 smithi125 bash[77299]: audit 2023-10-09T06:44:09.066221+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:44:09.220 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:09 smithi125 bash[77299]: cluster 2023-10-09T06:44:09.068526+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 7425/117 objects degraded (6346.154%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:09.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:09 smithi125 bash[80002]: audit 2023-10-09T06:44:09.066221+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:44:09.221 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:09 smithi125 bash[80002]: cluster 2023-10-09T06:44:09.068526+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 7425/117 objects degraded (6346.154%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:09.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:09.292 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:09 smithi155 bash[59495]: audit 2023-10-09T06:44:09.066221+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:44:09.293 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:09 smithi155 bash[59495]: cluster 2023-10-09T06:44:09.068526+0000 mon.a (mon.0) 1474 : cluster 3 Health check update: Degraded data redundancy: 7425/117 objects degraded (6346.154%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:09.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:09.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:09.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:09.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:09.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:09.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:09.684 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:09.707 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:09.811 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:09.835 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:09.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:09.973 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:09.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:10.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:10.108 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:10.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:10.347 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:10.365 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:10.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:10.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:10 smithi155 bash[59495]: cluster 2023-10-09T06:44:09.366088+0000 mgr.y (mgr.25723) 1187 : cluster 0 pgmap v1274: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 7425/84 objects degraded (8839.286%); 941 KiB/s, 154 objects/s recovering 2023-10-09T06:44:10.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:10 smithi155 bash[59495]: audit 2023-10-09T06:44:09.440457+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:10 smithi155 bash[59495]: audit 2023-10-09T06:44:09.446265+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.498 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:10.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:10.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[77299]: cluster 2023-10-09T06:44:09.366088+0000 mgr.y (mgr.25723) 1187 : cluster 0 pgmap v1274: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 7425/84 objects degraded (8839.286%); 941 KiB/s, 154 objects/s recovering 2023-10-09T06:44:10.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[77299]: audit 2023-10-09T06:44:09.440457+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[77299]: audit 2023-10-09T06:44:09.446265+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[80002]: cluster 2023-10-09T06:44:09.366088+0000 mgr.y (mgr.25723) 1187 : cluster 0 pgmap v1274: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 7425/84 objects degraded (8839.286%); 941 KiB/s, 154 objects/s recovering 2023-10-09T06:44:10.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[80002]: audit 2023-10-09T06:44:09.440457+0000 mon.a (mon.0) 1475 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:10 smithi125 bash[80002]: audit 2023-10-09T06:44:09.446265+0000 mon.a (mon.0) 1476 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:10.621 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:10.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:10.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:10.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:10.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:10.987 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:11.019 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:11.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:11.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:11.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:11.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:11.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:11.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:11.511 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.519 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:11] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-10-09T06:44:11.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:11.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:11.663 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:11.766 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.788 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:11.890 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:11.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:11.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:11.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:12.015 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:12.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:12.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:12.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:12.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:12.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:12.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:12 smithi125 bash[77299]: cluster 2023-10-09T06:44:11.366684+0000 mgr.y (mgr.25723) 1188 : cluster 0 pgmap v1275: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 7406/84 objects degraded (8816.667%); 355 KiB/s, 164 objects/s recovering 2023-10-09T06:44:12.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:12 smithi125 bash[80002]: cluster 2023-10-09T06:44:11.366684+0000 mgr.y (mgr.25723) 1188 : cluster 0 pgmap v1275: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 7406/84 objects degraded (8816.667%); 355 KiB/s, 164 objects/s recovering 2023-10-09T06:44:12.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:12 smithi155 bash[59495]: cluster 2023-10-09T06:44:11.366684+0000 mgr.y (mgr.25723) 1188 : cluster 0 pgmap v1275: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 7406/84 objects degraded (8816.667%); 355 KiB/s, 164 objects/s recovering 2023-10-09T06:44:12.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:12.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:12.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:12.839 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:12.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:12.942 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:12.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:12.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:13.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:13.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:13.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:13.208 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:13.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:13.374 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:13.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:13.505 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:13.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:13.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:13.739 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:13.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:13.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:14.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.033 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:14.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:14.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:14.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:14.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:14.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:14.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:14.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.554 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:14 smithi155 bash[59495]: cluster 2023-10-09T06:44:13.367446+0000 mgr.y (mgr.25723) 1189 : cluster 0 pgmap v1276: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 5 op/s; 7406/84 objects degraded (8816.667%); 128 KiB/s, 138 objects/s recovering 2023-10-09T06:44:14.554 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:14 smithi155 bash[59495]: cluster 2023-10-09T06:44:14.069561+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:14.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:14.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:14.711 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:14 smithi125 bash[77299]: cluster 2023-10-09T06:44:13.367446+0000 mgr.y (mgr.25723) 1189 : cluster 0 pgmap v1276: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 5 op/s; 7406/84 objects degraded (8816.667%); 128 KiB/s, 138 objects/s recovering 2023-10-09T06:44:14.711 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:14 smithi125 bash[77299]: cluster 2023-10-09T06:44:14.069561+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:14.711 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:14 smithi125 bash[80002]: cluster 2023-10-09T06:44:13.367446+0000 mgr.y (mgr.25723) 1189 : cluster 0 pgmap v1276: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 5 op/s; 7406/84 objects degraded (8816.667%); 128 KiB/s, 138 objects/s recovering 2023-10-09T06:44:14.712 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:14 smithi125 bash[80002]: cluster 2023-10-09T06:44:14.069561+0000 mon.a (mon.0) 1477 : cluster 3 Health check update: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:44:14.727 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:14.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:14.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:14.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:14.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:14.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:15.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:15.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:15.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:15.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:15.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:15.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:15.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:15.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:15.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:15.616 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:15.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:15.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:15.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:15.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:15.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:15.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:15.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:16.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:16.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:16.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:16.267 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:16.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:16.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:16.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[80002]: cluster 2023-10-09T06:44:15.368204+0000 mgr.y (mgr.25723) 1190 : cluster 0 pgmap v1277: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 104 KiB/s, 112 objects/s recovering 2023-10-09T06:44:16.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[80002]: audit 2023-10-09T06:44:15.549460+0000 mon.a (mon.0) 1478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.459 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[80002]: audit 2023-10-09T06:44:15.556740+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:16.553 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:16 smithi155 bash[59495]: cluster 2023-10-09T06:44:15.368204+0000 mgr.y (mgr.25723) 1190 : cluster 0 pgmap v1277: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 104 KiB/s, 112 objects/s recovering 2023-10-09T06:44:16.553 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:16 smithi155 bash[59495]: audit 2023-10-09T06:44:15.549460+0000 mon.a (mon.0) 1478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.553 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:16 smithi155 bash[59495]: audit 2023-10-09T06:44:15.556740+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.568 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:16.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:16.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:16.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.828 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:16.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[77299]: cluster 2023-10-09T06:44:15.368204+0000 mgr.y (mgr.25723) 1190 : cluster 0 pgmap v1277: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 104 KiB/s, 112 objects/s recovering 2023-10-09T06:44:16.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[77299]: audit 2023-10-09T06:44:15.549460+0000 mon.a (mon.0) 1478 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:16 smithi125 bash[77299]: audit 2023-10-09T06:44:15.556740+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:16.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:16.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:16.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:17.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.087 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:17.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:17.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:17.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:17.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:17.609 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:17.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:17.741 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:17.777 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:17.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:17.915 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:17.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:18.019 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:18.049 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:18.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:18.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:18.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:18.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:18.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:18.447 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:18.550 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:18.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:18.586 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:18 smithi125 bash[77299]: cluster 2023-10-09T06:44:17.368747+0000 mgr.y (mgr.25723) 1191 : cluster 0 pgmap v1278: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 95 KiB/s, 131 objects/s recovering 2023-10-09T06:44:18.587 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:18 smithi125 bash[80002]: cluster 2023-10-09T06:44:17.368747+0000 mgr.y (mgr.25723) 1191 : cluster 0 pgmap v1278: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 95 KiB/s, 131 objects/s recovering 2023-10-09T06:44:18.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:18.713 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:18.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:18.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:18 smithi155 bash[59495]: cluster 2023-10-09T06:44:17.368747+0000 mgr.y (mgr.25723) 1191 : cluster 0 pgmap v1278: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 7406/84 objects degraded (8816.667%); 95 KiB/s, 131 objects/s recovering 2023-10-09T06:44:18.849 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:18.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:18.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:18.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:19.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:19.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:19.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:19.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:19.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:19.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:19.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:19.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:19.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:19.590 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:19 smithi125 bash[80002]: audit 2023-10-09T06:44:18.604635+0000 mon.a (mon.0) 1480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:19.591 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:19 smithi125 bash[77299]: audit 2023-10-09T06:44:18.604635+0000 mon.a (mon.0) 1480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:19.607 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.631 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:19 smithi155 bash[59495]: audit 2023-10-09T06:44:18.604635+0000 mon.a (mon.0) 1480 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:19.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.762 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:19.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:19.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:19.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.024 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:20.034 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:20.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:20.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:20.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:20.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.269 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:20.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:20.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:20.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:20 smithi125 bash[77299]: cluster 2023-10-09T06:44:19.369315+0000 mgr.y (mgr.25723) 1192 : cluster 0 pgmap v1279: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 7406/84 objects degraded (8816.667%); 0 B/s, 106 objects/s recovering 2023-10-09T06:44:20.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:20 smithi125 bash[80002]: cluster 2023-10-09T06:44:19.369315+0000 mgr.y (mgr.25723) 1192 : cluster 0 pgmap v1279: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 7406/84 objects degraded (8816.667%); 0 B/s, 106 objects/s recovering 2023-10-09T06:44:20.662 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:20.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:20.795 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:20 smithi155 bash[59495]: cluster 2023-10-09T06:44:19.369315+0000 mgr.y (mgr.25723) 1192 : cluster 0 pgmap v1279: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 7406/84 objects degraded (8816.667%); 0 B/s, 106 objects/s recovering 2023-10-09T06:44:20.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.930 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:20.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:21.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:21.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:21.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:21.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:21.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:21.318 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:21.388 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:21] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T06:44:21.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:21.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:21.530 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:21.531 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:21.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:21.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:21.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:21.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:21.721 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:21.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:21.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:21.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:21.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:21.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:22.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:22.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:22.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:22.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:22.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:22.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:22.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:22.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:22.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:22 smithi125 bash[77299]: cluster 2023-10-09T06:44:21.369936+0000 mgr.y (mgr.25723) 1193 : cluster 0 pgmap v1280: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 78 objects/s recovering 2023-10-09T06:44:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:22 smithi125 bash[80002]: cluster 2023-10-09T06:44:21.369936+0000 mgr.y (mgr.25723) 1193 : cluster 0 pgmap v1280: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 78 objects/s recovering 2023-10-09T06:44:22.640 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:22 smithi155 bash[59495]: cluster 2023-10-09T06:44:21.369936+0000 mgr.y (mgr.25723) 1193 : cluster 0 pgmap v1280: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 78 objects/s recovering 2023-10-09T06:44:22.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:22.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:22.788 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:22.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:22.916 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:22.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:23.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:23.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:23.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:23.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:23.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:23.410 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:23.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:23.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:23.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:23.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:23.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:23.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:23.715 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:23.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:23.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:23.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.981 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:23.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:24.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:24.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:24.220 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:24.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:24.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:24.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:24.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:24.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:24.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:24.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:24 smithi155 bash[59495]: cluster 2023-10-09T06:44:23.370378+0000 mgr.y (mgr.25723) 1194 : cluster 0 pgmap v1281: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:24.653 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:24.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:24.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:24.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:24 smithi125 bash[77299]: cluster 2023-10-09T06:44:23.370378+0000 mgr.y (mgr.25723) 1194 : cluster 0 pgmap v1281: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:24.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:24 smithi125 bash[80002]: cluster 2023-10-09T06:44:23.370378+0000 mgr.y (mgr.25723) 1194 : cluster 0 pgmap v1281: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:24.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:24.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:24.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:24.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:25.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.055 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:25.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:25.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:25.281 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.314 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:25.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:25.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:25.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:25.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:44:25.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:25.745 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:25.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:25.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:25.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:25.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:25.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:26.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:26.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:26.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:26.391 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:26.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:26.526 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:26.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:26.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:26 smithi125 bash[77299]: cluster 2023-10-09T06:44:25.371044+0000 mgr.y (mgr.25723) 1195 : cluster 0 pgmap v1282: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:26.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:26 smithi125 bash[80002]: cluster 2023-10-09T06:44:25.371044+0000 mgr.y (mgr.25723) 1195 : cluster 0 pgmap v1282: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:26.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:26.815 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:26.828 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:26 smithi155 bash[59495]: cluster 2023-10-09T06:44:25.371044+0000 mgr.y (mgr.25723) 1195 : cluster 0 pgmap v1282: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 55 objects/s recovering 2023-10-09T06:44:26.846 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:26.950 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:26.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:26.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:27.071 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:27.094 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:27.197 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:27.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:27.333 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:27.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:27.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:27.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:27.586 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:27.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:27.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:27.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:27.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:27.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:27.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:27.985 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:28.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:28.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:28.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:44:28.232 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:44:28.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:28.258 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:44:28.282 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:44:28.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:28.305 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:44:28.333 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:44:28.359 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:44:28.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:28.389 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:44:28.390 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.390 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.390 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.390 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.390 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.391 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.391 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.391 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.406 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:28.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:28.509 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:28.519 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: cluster 2023-10-09T06:44:27.371753+0000 mgr.y (mgr.25723) 1196 : cluster 0 pgmap v1283: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 86 objects/s recovering 2023-10-09T06:44:28.519 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:27.403156+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.519 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:27.410750+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.520 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:27.880532+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:44:28.520 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:27.881752+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:44:28.520 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:27.889464+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.520 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:28 smithi155 bash[59495]: audit 2023-10-09T06:44:28.186017+0000 mon.a (mon.0) 1486 : audit 0 from='client.? 172.21.15.125:0/3341251437' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:28.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:28.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:28.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.806 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:28.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: cluster 2023-10-09T06:44:27.371753+0000 mgr.y (mgr.25723) 1196 : cluster 0 pgmap v1283: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 86 objects/s recovering 2023-10-09T06:44:28.870 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:27.403156+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:27.410750+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:27.880532+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:27.881752+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:27.889464+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[77299]: audit 2023-10-09T06:44:28.186017+0000 mon.a (mon.0) 1486 : audit 0 from='client.? 172.21.15.125:0/3341251437' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: cluster 2023-10-09T06:44:27.371753+0000 mgr.y (mgr.25723) 1196 : cluster 0 pgmap v1283: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 86 objects/s recovering 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:27.403156+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.871 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:27.410750+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:27.880532+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:44:28.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:27.881752+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:44:28.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:27.889464+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:28.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:28 smithi125 bash[80002]: audit 2023-10-09T06:44:28.186017+0000 mon.a (mon.0) 1486 : audit 0 from='client.? 172.21.15.125:0/3341251437' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:28.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.941 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:28.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:29.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:29.072 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:29.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:29.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:29.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:29.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:29.460 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:29.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:29.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:29.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:29.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:29.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:30.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:30.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:30.227 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:30.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:30.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:30.371 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:30 smithi155 bash[59495]: cluster 2023-10-09T06:44:29.372573+0000 mgr.y (mgr.25723) 1197 : cluster 0 pgmap v1284: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 60 objects/s recovering 2023-10-09T06:44:30.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:30 smithi125 bash[77299]: cluster 2023-10-09T06:44:29.372573+0000 mgr.y (mgr.25723) 1197 : cluster 0 pgmap v1284: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 60 objects/s recovering 2023-10-09T06:44:30.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:30 smithi125 bash[80002]: cluster 2023-10-09T06:44:29.372573+0000 mgr.y (mgr.25723) 1197 : cluster 0 pgmap v1284: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 60 objects/s recovering 2023-10-09T06:44:30.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:30.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:30.519 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:30.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:30.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:30.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:30.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:30.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:30.778 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:30.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:30.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:30.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:30.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:31.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:31.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:31.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:31.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:31.272 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:31] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.33.4" 2023-10-09T06:44:31.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:31.334 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:31.358 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.0 primary-affinity to 0 (802) 2023-10-09T06:44:31.437 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:31.444 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:31 smithi125 bash[77299]: audit 2023-10-09T06:44:31.279176+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.125:0/4146725285' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.445 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:31 smithi125 bash[77299]: audit 2023-10-09T06:44:31.279603+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.445 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:31 smithi125 bash[80002]: audit 2023-10-09T06:44:31.279176+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.125:0/4146725285' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.445 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:31 smithi125 bash[80002]: audit 2023-10-09T06:44:31.279603+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:31.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:31.572 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:31 smithi155 bash[59495]: audit 2023-10-09T06:44:31.279176+0000 mon.c (mon.2) 111 : audit 1 from='client.? 172.21.15.125:0/4146725285' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.572 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:31 smithi155 bash[59495]: audit 2023-10-09T06:44:31.279603+0000 mon.a (mon.0) 1487 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-09T06:44:31.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:31.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:31.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:31.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:31.813 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:31.844 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:31.948 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:31.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:31.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:32.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:32.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:32.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:32.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:32.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:32.350 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:44:32.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:32.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:32.464 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:32.500 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:32.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: audit 2023-10-09T06:44:31.339553+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:31.339618+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e281: 8 total, 8 up, 6 in 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:31.339827+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:31.343291+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:31.343873+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:31.373092+0000 mgr.y (mgr.25723) 1198 : cluster 0 pgmap v1286: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 76 objects/s recovering 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: audit 2023-10-09T06:44:31.452789+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.331835+0000 mon.a (mon.0) 1491 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: audit 2023-10-09T06:44:32.339608+0000 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:32.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.339734+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e282: 8 total, 8 up, 6 in 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: audit 2023-10-09T06:44:31.339553+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:31.339618+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e281: 8 total, 8 up, 6 in 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:31.339827+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:31.343291+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:31.343873+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:31.373092+0000 mgr.y (mgr.25723) 1198 : cluster 0 pgmap v1286: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 76 objects/s recovering 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: audit 2023-10-09T06:44:31.452789+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.331835+0000 mon.a (mon.0) 1491 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:32.612 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: audit 2023-10-09T06:44:32.339608+0000 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:32.613 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:32 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.339734+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e282: 8 total, 8 up, 6 in 2023-10-09T06:44:32.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:32.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: audit 2023-10-09T06:44:31.339553+0000 mon.a (mon.0) 1488 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-09T06:44:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:31.339618+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e281: 8 total, 8 up, 6 in 2023-10-09T06:44:32.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:31.339827+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:31.343291+0000 osd.4 (osd.4) 238 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:31.343873+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e281 with expected crc 2023-10-09T06:44:32.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:31.373092+0000 mgr.y (mgr.25723) 1198 : cluster 0 pgmap v1286: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 76 objects/s recovering 2023-10-09T06:44:32.647 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: audit 2023-10-09T06:44:31.452789+0000 mon.a (mon.0) 1490 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:32.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.331835+0000 mon.a (mon.0) 1491 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:32.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: audit 2023-10-09T06:44:32.339608+0000 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.125:0/2317177559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:32.648 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:32 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.339734+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e282: 8 total, 8 up, 6 in 2023-10-09T06:44:32.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:32.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:32.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:32.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:32.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:32.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:32.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:33.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:33.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:33.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:33.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:33.276 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:33.380 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:33 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.337800+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:33 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.339490+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:33 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.341609+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:33 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.343364+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:33 smithi155 bash[59495]: cluster 2023-10-09T06:44:32.347308+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.400 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:33.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:33.533 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:33.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:33.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:33.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.337800+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.339490+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.341609+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.343364+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[77299]: cluster 2023-10-09T06:44:32.347308+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.337800+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.339490+0000 osd.4 (osd.4) 239 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.341609+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.343364+0000 osd.4 (osd.4) 240 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.677 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:33 smithi125 bash[80002]: cluster 2023-10-09T06:44:32.347308+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e282 with expected crc 2023-10-09T06:44:33.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:33.796 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:33.899 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:33.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:34.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:34.053 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:34.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:34.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:34.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:34.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:34.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:34.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:34.568 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[77299]: cluster 2023-10-09T06:44:33.373527+0000 mgr.y (mgr.25723) 1199 : cluster 0 pgmap v1288: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 95 objects/s recovering 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[77299]: audit 2023-10-09T06:44:33.608942+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[77299]: audit 2023-10-09T06:44:33.609841+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[80002]: cluster 2023-10-09T06:44:33.373527+0000 mgr.y (mgr.25723) 1199 : cluster 0 pgmap v1288: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 95 objects/s recovering 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[80002]: audit 2023-10-09T06:44:33.608942+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:34.580 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:34 smithi125 bash[80002]: audit 2023-10-09T06:44:33.609841+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:34.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:34 smithi155 bash[59495]: cluster 2023-10-09T06:44:33.373527+0000 mgr.y (mgr.25723) 1199 : cluster 0 pgmap v1288: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 95 objects/s recovering 2023-10-09T06:44:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:34 smithi155 bash[59495]: audit 2023-10-09T06:44:33.608942+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:34 smithi155 bash[59495]: audit 2023-10-09T06:44:33.609841+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:34.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:34.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:34.833 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:34.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:34.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:34.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:34.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:35.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:35.131 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:35.234 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:35.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:35.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:35.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:35.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:35.640 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:35.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:35.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:44:35.804 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:35.907 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:35.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:36.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:36.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:36.171 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:36.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:36.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:36.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:36.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:36.421 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:36.558 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:36.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:36 smithi125 bash[80002]: cluster 2023-10-09T06:44:35.374417+0000 mgr.y (mgr.25723) 1200 : cluster 0 pgmap v1289: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 45 objects/s recovering 2023-10-09T06:44:36.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:36 smithi125 bash[77299]: cluster 2023-10-09T06:44:35.374417+0000 mgr.y (mgr.25723) 1200 : cluster 0 pgmap v1289: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 45 objects/s recovering 2023-10-09T06:44:36.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.705 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:36 smithi155 bash[59495]: cluster 2023-10-09T06:44:35.374417+0000 mgr.y (mgr.25723) 1200 : cluster 0 pgmap v1289: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 45 objects/s recovering 2023-10-09T06:44:36.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:36.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:36.849 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:36.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:36.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:36.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:37.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:37.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:37.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.244 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:37.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:37.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.512 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:37.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:37.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:37.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:37.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:37.927 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:37.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:37.949 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:38.031 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:38.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:38.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:38.318 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:38.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:38.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:38.559 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:38.567 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:38 smithi125 bash[77299]: cluster 2023-10-09T06:44:37.374941+0000 mgr.y (mgr.25723) 1201 : cluster 0 pgmap v1290: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 23 objects/s recovering 2023-10-09T06:44:38.568 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:38 smithi125 bash[80002]: cluster 2023-10-09T06:44:37.374941+0000 mgr.y (mgr.25723) 1201 : cluster 0 pgmap v1290: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 23 objects/s recovering 2023-10-09T06:44:38.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:38.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:38.704 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:38 smithi155 bash[59495]: cluster 2023-10-09T06:44:37.374941+0000 mgr.y (mgr.25723) 1201 : cluster 0 pgmap v1290: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 23 objects/s recovering 2023-10-09T06:44:38.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:38.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:38.961 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:38.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:38.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:39.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:39.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:39.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:39.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:39.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:39.394 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:39.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:39.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:44:39.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:39.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:39.764 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:39.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:39.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:39.893 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:40.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:40.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:40.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:40.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:40.273 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:40.293 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:40.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:40 smithi125 bash[80002]: cluster 2023-10-09T06:44:39.375779+0000 mgr.y (mgr.25723) 1202 : cluster 0 pgmap v1291: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%) 2023-10-09T06:44:40.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:40 smithi125 bash[77299]: cluster 2023-10-09T06:44:39.375779+0000 mgr.y (mgr.25723) 1202 : cluster 0 pgmap v1291: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%) 2023-10-09T06:44:40.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:40 smithi155 bash[59495]: cluster 2023-10-09T06:44:39.375779+0000 mgr.y (mgr.25723) 1202 : cluster 0 pgmap v1291: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%) 2023-10-09T06:44:40.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:40.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:40.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:40.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:40.652 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:40.684 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:40.789 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:40.813 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:40.915 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:40.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:40.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:41.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:41.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:41.179 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:41.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:41.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:41.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:41.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:41.444 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:41.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:41.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:41.604 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:41.612 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:41] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-10-09T06:44:41.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:41.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:41.836 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:41.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:41.962 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:41.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:41.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:42.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:42.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:42.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:42.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:42.268 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:42.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:42.490 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:42.521 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:42.625 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:42 smithi125 bash[77299]: cluster 2023-10-09T06:44:41.377101+0000 mgr.y (mgr.25723) 1203 : cluster 0 pgmap v1292: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 24 objects/s recovering 2023-10-09T06:44:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:42 smithi125 bash[80002]: cluster 2023-10-09T06:44:41.377101+0000 mgr.y (mgr.25723) 1203 : cluster 0 pgmap v1292: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 24 objects/s recovering 2023-10-09T06:44:42.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:42 smithi155 bash[59495]: cluster 2023-10-09T06:44:41.377101+0000 mgr.y (mgr.25723) 1203 : cluster 0 pgmap v1292: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 24 objects/s recovering 2023-10-09T06:44:42.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:42.746 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 7, 6] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:44:42.746 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:44:42.747 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 2, 3, 4, 7, 6] 2023-10-09T06:44:42.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T06:44:42.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:42.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:42.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:42.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:43.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:43.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:43.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:43.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:43.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:43.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:43.417 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:43 smithi125 bash[80002]: audit 2023-10-09T06:44:42.682270+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.125:0/2843748357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.444 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:43 smithi125 bash[80002]: audit 2023-10-09T06:44:42.682725+0000 mon.a (mon.0) 1496 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.451 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:43.460 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_4' removed 2023-10-09T06:44:43.555 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:43.583 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:43.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.698 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:43 smithi155 bash[59495]: audit 2023-10-09T06:44:42.682270+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.125:0/2843748357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.698 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:43 smithi155 bash[59495]: audit 2023-10-09T06:44:42.682725+0000 mon.a (mon.0) 1496 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:43.783 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:43 smithi125 bash[77299]: audit 2023-10-09T06:44:42.682270+0000 mon.c (mon.2) 112 : audit 1 from='client.? 172.21.15.125:0/2843748357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.783 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:43 smithi125 bash[77299]: audit 2023-10-09T06:44:42.682725+0000 mon.a (mon.0) 1496 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:44:43.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.852 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:43.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:43.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:43.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:43.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:43.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:44.097 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:44.132 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:44.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:44.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:44.405 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:44.540 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:44.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.377591+0000 mgr.y (mgr.25723) 1204 : cluster 0 pgmap v1293: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 22 objects/s recovering 2023-10-09T06:44:44.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: audit 2023-10-09T06:44:43.443657+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.443760+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e283: 8 total, 8 up, 6 in 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.914845+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.917353+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.919901+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.961437+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.377591+0000 mgr.y (mgr.25723) 1204 : cluster 0 pgmap v1293: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 22 objects/s recovering 2023-10-09T06:44:44.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: audit 2023-10-09T06:44:43.443657+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:44:44.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.443760+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e283: 8 total, 8 up, 6 in 2023-10-09T06:44:44.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.914845+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.917353+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.919901+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:44 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.961437+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:44.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:44.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.690 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.377591+0000 mgr.y (mgr.25723) 1204 : cluster 0 pgmap v1293: 89 pgs: 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 7406/84 objects degraded (8816.667%); 0 B/s, 22 objects/s recovering 2023-10-09T06:44:44.690 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: audit 2023-10-09T06:44:43.443657+0000 mon.a (mon.0) 1497 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:44:44.690 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.443760+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e283: 8 total, 8 up, 6 in 2023-10-09T06:44:44.690 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.914845+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.690 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.917353+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.919901+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:44 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.961437+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:44.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:44.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:44.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:44.984 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.014 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:45.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.146 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:45.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:45.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:45.414 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:45.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.819382+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.527 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.821897+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.528 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[77299]: cluster 2023-10-09T06:44:43.961392+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.819382+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.821897+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.528 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:45 smithi125 bash[80002]: cluster 2023-10-09T06:44:43.961392+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.545 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:45.648 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:45.676 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:45.695 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:44:45.696 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:44:45.696 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T06:44:45.696 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T06:44:45.781 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.794 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:45 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.819382+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.794 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:45 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.821897+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.794 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:45 smithi155 bash[59495]: cluster 2023-10-09T06:44:43.961392+0000 osd.4 (osd.4) 241 : cluster 3 failed to encode map e283 with expected crc 2023-10-09T06:44:45.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:45.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.951 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:45.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:46.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:46.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:46.173 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:46.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:46.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:46.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:46.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:46.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:46.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:46.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:46.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:46 smithi155 bash[59495]: cluster 2023-10-09T06:44:45.378296+0000 mgr.y (mgr.25723) 1205 : cluster 0 pgmap v1295: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:46.599 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:46 smithi155 bash[59495]: cluster 2023-10-09T06:44:45.440849+0000 mon.a (mon.0) 1499 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:44:46.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:46.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:46.722 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:46 smithi125 bash[77299]: cluster 2023-10-09T06:44:45.378296+0000 mgr.y (mgr.25723) 1205 : cluster 0 pgmap v1295: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:46.722 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:46 smithi125 bash[77299]: cluster 2023-10-09T06:44:45.440849+0000 mon.a (mon.0) 1499 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:44:46.722 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:46 smithi125 bash[80002]: cluster 2023-10-09T06:44:45.378296+0000 mgr.y (mgr.25723) 1205 : cluster 0 pgmap v1295: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:46.722 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:46 smithi125 bash[80002]: cluster 2023-10-09T06:44:45.440849+0000 mon.a (mon.0) 1499 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7406/84 objects degraded (8816.667%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:44:46.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:46.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:46.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:46.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:46.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:46.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:47.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:47.117 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:47.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:47.254 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:47.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:47.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:47.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:47.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:47.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:47.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:47.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:47.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:47.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.834 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:47.938 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:47.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:48.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.093 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:48.197 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.225 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:48.331 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:48.361 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:48.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:48.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:48.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:48.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:48 smithi125 bash[77299]: cluster 2023-10-09T06:44:47.379082+0000 mgr.y (mgr.25723) 1206 : cluster 0 pgmap v1296: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:48.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:48 smithi125 bash[80002]: cluster 2023-10-09T06:44:47.379082+0000 mgr.y (mgr.25723) 1206 : cluster 0 pgmap v1296: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:48.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:48.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:48 smithi155 bash[59495]: cluster 2023-10-09T06:44:47.379082+0000 mgr.y (mgr.25723) 1206 : cluster 0 pgmap v1296: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:48.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:48.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:48.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:48.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:48.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:48.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:48.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:49.029 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:49.132 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:49.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:49.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:49.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:49.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:49.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:49.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:49.572 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:49.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:49.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:49.813 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:49.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[77299]: audit 2023-10-09T06:44:48.610548+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[77299]: audit 2023-10-09T06:44:48.611671+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:49.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[77299]: audit 2023-10-09T06:44:49.303425+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[77299]: cluster 2023-10-09T06:44:49.379613+0000 mgr.y (mgr.25723) 1207 : cluster 0 pgmap v1297: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:49.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[80002]: audit 2023-10-09T06:44:48.610548+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[80002]: audit 2023-10-09T06:44:48.611671+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:49.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[80002]: audit 2023-10-09T06:44:49.303425+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:49 smithi125 bash[80002]: cluster 2023-10-09T06:44:49.379613+0000 mgr.y (mgr.25723) 1207 : cluster 0 pgmap v1297: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:49.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:49 smithi155 bash[59495]: audit 2023-10-09T06:44:48.610548+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:49 smithi155 bash[59495]: audit 2023-10-09T06:44:48.611671+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:49 smithi155 bash[59495]: audit 2023-10-09T06:44:49.303425+0000 mon.a (mon.0) 1502 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:44:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:49 smithi155 bash[59495]: cluster 2023-10-09T06:44:49.379613+0000 mgr.y (mgr.25723) 1207 : cluster 0 pgmap v1297: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:49.903 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:49.951 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:49.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:49.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:50.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:50.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:50.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:50.212 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:50.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:50.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:50.370 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:50.381 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:44:50.381 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:44:50.381 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:44:50.381 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:44:50.381 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:44:50.474 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:50.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:50.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:50.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:50.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:50.743 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:50.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:50.880 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:50.913 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:50.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:51.017 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:51.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:51.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:51.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:51.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:51.293 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:44:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:44:51] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.33.4" 2023-10-09T06:44:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:51.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:51.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:51.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.590 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:51.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:51.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:51.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:51.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:51.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:51.954 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:51.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:52.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:52.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:52.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:52.255 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:52.358 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:52.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:52.486 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:52.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:52.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:52.634 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:52 smithi125 bash[77299]: cluster 2023-10-09T06:44:51.380695+0000 mgr.y (mgr.25723) 1208 : cluster 0 pgmap v1298: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:52.635 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:52 smithi125 bash[80002]: cluster 2023-10-09T06:44:51.380695+0000 mgr.y (mgr.25723) 1208 : cluster 0 pgmap v1298: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:52.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:52.751 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:52.764 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:52 smithi155 bash[59495]: cluster 2023-10-09T06:44:51.380695+0000 mgr.y (mgr.25723) 1208 : cluster 0 pgmap v1298: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:52.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:52.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:52.912 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:52.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:53.015 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:53.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:53.143 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:53.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:53.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:53.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:53.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:53.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:53.534 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:53.571 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:53.675 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:53.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:53.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:53.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:53.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:53.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:53.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:54.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:54.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:54.196 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:44:54.206 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:44:54.207 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:54.226 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:44:54.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:54.246 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:44:54.268 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:44:54.296 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:44:54.321 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:44:54.342 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:54.347 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:44:54.373 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.375 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:44:54.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:54.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:54.503 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:54.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:54.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:54 smithi155 bash[59495]: cluster 2023-10-09T06:44:53.381355+0000 mgr.y (mgr.25723) 1209 : cluster 0 pgmap v1299: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:54.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:54 smithi155 bash[59495]: audit 2023-10-09T06:44:54.177157+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.125:0/1983503622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:54.667 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:54.693 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:54 smithi125 bash[80002]: cluster 2023-10-09T06:44:53.381355+0000 mgr.y (mgr.25723) 1209 : cluster 0 pgmap v1299: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:54.693 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:54 smithi125 bash[80002]: audit 2023-10-09T06:44:54.177157+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.125:0/1983503622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:54.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:54 smithi125 bash[77299]: cluster 2023-10-09T06:44:53.381355+0000 mgr.y (mgr.25723) 1209 : cluster 0 pgmap v1299: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:54.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:54 smithi125 bash[77299]: audit 2023-10-09T06:44:54.177157+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.125:0/1983503622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:54.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:54.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:54.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:54.884 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:54.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:54.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:55.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:55.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:55.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:55.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.276 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:55.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:55.504 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:55.526 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:55.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:55.658 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:55.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:55.892 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:55.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:55.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:56.022 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:56.160 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:56.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:56.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:56.435 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:56.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:56.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:56.600 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:56 smithi155 bash[59495]: cluster 2023-10-09T06:44:55.382116+0000 mgr.y (mgr.25723) 1210 : cluster 0 pgmap v1300: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:56.703 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:56.715 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:56 smithi125 bash[77299]: cluster 2023-10-09T06:44:55.382116+0000 mgr.y (mgr.25723) 1210 : cluster 0 pgmap v1300: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:56.715 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:56 smithi125 bash[80002]: cluster 2023-10-09T06:44:55.382116+0000 mgr.y (mgr.25723) 1210 : cluster 0 pgmap v1300: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:56.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:56.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:56.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:56.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:56.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:57.007 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:57.111 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:57.134 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:57.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:57.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:57.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:57.369 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:57.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:57.494 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:57.520 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:57.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:57.622 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:57.649 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:57.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:57.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:57.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:57.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:57.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:57.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:58.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.058 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:58.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.191 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:58.295 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:58.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.442 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:58.545 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.576 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:58.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:44:58 smithi125 bash[77299]: cluster 2023-10-09T06:44:57.382717+0000 mgr.y (mgr.25723) 1211 : cluster 0 pgmap v1301: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:44:58 smithi125 bash[80002]: cluster 2023-10-09T06:44:57.382717+0000 mgr.y (mgr.25723) 1211 : cluster 0 pgmap v1301: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:58.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:58.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:58.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:58.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:58.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:44:58 smithi155 bash[59495]: cluster 2023-10-09T06:44:57.382717+0000 mgr.y (mgr.25723) 1211 : cluster 0 pgmap v1301: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:44:58.953 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:58.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:58.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:59.086 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:59.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:59.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:59.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:59.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:59.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:59.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:59.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:59.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.646 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:59.748 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:59.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:59.880 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:59.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:00.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:00.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:00.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:00.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:00.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:00 smithi125 bash[77299]: cluster 2023-10-09T06:44:59.383146+0000 mgr.y (mgr.25723) 1212 : cluster 0 pgmap v1302: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:00 smithi125 bash[80002]: cluster 2023-10-09T06:44:59.383146+0000 mgr.y (mgr.25723) 1212 : cluster 0 pgmap v1302: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:00.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:00 smithi155 bash[59495]: cluster 2023-10-09T06:44:59.383146+0000 mgr.y (mgr.25723) 1212 : cluster 0 pgmap v1302: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:00.407 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:00.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:00.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:00.565 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:00.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:00.692 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:00.795 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:00.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:00.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:45:00.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:00.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:01.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:45:01.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.217 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:01 smithi125 bash[80002]: audit 2023-10-09T06:45:00.385944+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.125:0/390600959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.218 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:01 smithi125 bash[80002]: audit 2023-10-09T06:45:00.386422+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:01 smithi125 bash[77299]: audit 2023-10-09T06:45:00.385944+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.125:0/390600959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.218 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:01 smithi125 bash[77299]: audit 2023-10-09T06:45:00.386422+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:01.344 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:01.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:01.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:01 smithi155 bash[59495]: audit 2023-10-09T06:45:00.385944+0000 mon.c (mon.2) 114 : audit 1 from='client.? 172.21.15.125:0/390600959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:01 smithi155 bash[59495]: audit 2023-10-09T06:45:00.386422+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:01.472 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.507 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:01.548 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:01] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.33.4" 2023-10-09T06:45:01.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.644 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:01.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:01.775 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:01.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:01.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:01.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:02.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:02.039 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:02.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:02.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:02 smithi155 bash[59495]: cluster 2023-10-09T06:45:01.090180+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:02 smithi155 bash[59495]: audit 2023-10-09T06:45:01.101929+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:02 smithi155 bash[59495]: cluster 2023-10-09T06:45:01.102009+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e284: 8 total, 8 up, 6 in 2023-10-09T06:45:02.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:02 smithi155 bash[59495]: cluster 2023-10-09T06:45:01.383510+0000 mgr.y (mgr.25723) 1213 : cluster 0 pgmap v1304: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:02.169 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:02.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:02.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:02.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:02.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:02.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[77299]: cluster 2023-10-09T06:45:01.090180+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:02.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[77299]: audit 2023-10-09T06:45:01.101929+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:02.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[77299]: cluster 2023-10-09T06:45:01.102009+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e284: 8 total, 8 up, 6 in 2023-10-09T06:45:02.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[77299]: cluster 2023-10-09T06:45:01.383510+0000 mgr.y (mgr.25723) 1213 : cluster 0 pgmap v1304: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:02.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[80002]: cluster 2023-10-09T06:45:01.090180+0000 mon.a (mon.0) 1504 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:02.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[80002]: audit 2023-10-09T06:45:01.101929+0000 mon.a (mon.0) 1505 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:02.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[80002]: cluster 2023-10-09T06:45:01.102009+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e284: 8 total, 8 up, 6 in 2023-10-09T06:45:02.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:02 smithi125 bash[80002]: cluster 2023-10-09T06:45:01.383510+0000 mgr.y (mgr.25723) 1213 : cluster 0 pgmap v1304: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:02.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:02.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:02.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.583 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:02.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:02.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:02.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:02.821 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:02.956 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:02.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:03.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:03.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:03.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:03.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:03 smithi155 bash[59495]: cluster 2023-10-09T06:45:01.968424+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:03 smithi155 bash[59495]: cluster 2023-10-09T06:45:01.970963+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:03 smithi155 bash[59495]: cluster 2023-10-09T06:45:02.491116+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:03 smithi155 bash[59495]: cluster 2023-10-09T06:45:02.491120+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.236 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:03 smithi155 bash[59495]: cluster 2023-10-09T06:45:02.491137+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[80002]: cluster 2023-10-09T06:45:01.968424+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[80002]: cluster 2023-10-09T06:45:01.970963+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[80002]: cluster 2023-10-09T06:45:02.491116+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[80002]: cluster 2023-10-09T06:45:02.491120+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[80002]: cluster 2023-10-09T06:45:02.491137+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.337 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: cluster 2023-10-09T06:45:01.968424+0000 osd.4 (osd.4) 242 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: cluster 2023-10-09T06:45:01.970963+0000 osd.4 (osd.4) 243 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: cluster 2023-10-09T06:45:02.491116+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: cluster 2023-10-09T06:45:02.491120+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.338 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: cluster 2023-10-09T06:45:02.491137+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e284 with expected crc 2023-10-09T06:45:03.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:03.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:03.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:03.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:03.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:03.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:03 smithi125 bash[77299]: debug 2023-10-09T06:45:03.482+0000 7ff949fd9700 -1 mon.a@0(leader).osd e284 definitely_dead 0 2023-10-09T06:45:03.651 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:03.755 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:03.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:03.879 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:03.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:03.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:03.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:04.018 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:04.133 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.2. 2023-10-09T06:45:04.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:04 smithi155 bash[59495]: cluster 2023-10-09T06:45:03.384073+0000 mgr.y (mgr.25723) 1214 : cluster 0 pgmap v1305: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:04.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:04 smithi155 bash[59495]: audit 2023-10-09T06:45:03.488962+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:45:04.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:04 smithi155 bash[59495]: cluster 2023-10-09T06:45:03.489087+0000 mon.a (mon.0) 1508 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:45:04.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:04 smithi155 bash[59495]: audit 2023-10-09T06:45:03.605676+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:04.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:04.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:04.291 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:04.301 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[80002]: cluster 2023-10-09T06:45:03.384073+0000 mgr.y (mgr.25723) 1214 : cluster 0 pgmap v1305: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:04.301 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[80002]: audit 2023-10-09T06:45:03.488962+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:45:04.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[80002]: cluster 2023-10-09T06:45:03.489087+0000 mon.a (mon.0) 1508 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:45:04.302 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[80002]: audit 2023-10-09T06:45:03.605676+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:04.302 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[77299]: cluster 2023-10-09T06:45:03.384073+0000 mgr.y (mgr.25723) 1214 : cluster 0 pgmap v1305: 73 pgs: 73 active+clean; 4.6 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:45:04.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[77299]: audit 2023-10-09T06:45:03.488962+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T06:45:04.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[77299]: cluster 2023-10-09T06:45:03.489087+0000 mon.a (mon.0) 1508 : cluster 1 Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T06:45:04.303 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:04 smithi125 bash[77299]: audit 2023-10-09T06:45:03.605676+0000 mon.a (mon.0) 1509 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:04.319 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:04.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:04.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:04.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:04.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:04.694 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:04.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:04.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:04.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:04.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:04.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:04.982 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:04.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:05.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:05.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:05.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: audit 2023-10-09T06:45:04.115892+0000 mon.a (mon.0) 1510 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:45:05.126 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.115989+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e285: 8 total, 8 up, 5 in 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.117290+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.117497+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.118016+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.119687+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.121938+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.127 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:05 smithi155 bash[59495]: cluster 2023-10-09T06:45:04.123242+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:05.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:05.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:45:05.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: audit 2023-10-09T06:45:04.115892+0000 mon.a (mon.0) 1510 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.115989+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e285: 8 total, 8 up, 5 in 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.117290+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.117497+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.118016+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.119687+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.121938+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[80002]: cluster 2023-10-09T06:45:04.123242+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: audit 2023-10-09T06:45:04.115892+0000 mon.a (mon.0) 1510 : audit 1 from='client.? 172.21.15.125:0/2579001189' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.115989+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e285: 8 total, 8 up, 5 in 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.117290+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.117497+0000 osd.4 (osd.4) 244 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.118016+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.119687+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.121938+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:05 smithi125 bash[77299]: cluster 2023-10-09T06:45:04.123242+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e285 with expected crc 2023-10-09T06:45:05.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:05.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:05.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:05.639 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:05.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:05.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:05.803 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:05.906 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:05.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:05.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:06.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:06.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:06.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:06.186 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.115689+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.115868+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.116057+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e286: 8 total, 8 up, 5 in 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.118737+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.118992+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.119144+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.121960+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.187 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:06 smithi155 bash[59495]: cluster 2023-10-09T06:45:05.384559+0000 mgr.y (mgr.25723) 1215 : cluster 0 pgmap v1308: 73 pgs: 18 peering, 55 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 170 B/s rd, 0 op/s; 184 B/s, 0 objects/s recovering 2023-10-09T06:45:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:06.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:06.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.115689+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.115868+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.116057+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e286: 8 total, 8 up, 5 in 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.118737+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.118992+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.119144+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.121960+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.323 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[77299]: cluster 2023-10-09T06:45:05.384559+0000 mgr.y (mgr.25723) 1215 : cluster 0 pgmap v1308: 73 pgs: 18 peering, 55 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 170 B/s rd, 0 op/s; 184 B/s, 0 objects/s recovering 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.115689+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.115868+0000 osd.4 (osd.4) 245 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.116057+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e286: 8 total, 8 up, 5 in 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.118737+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.118992+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.119144+0000 osd.4 (osd.4) 246 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.121960+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e286 with expected crc 2023-10-09T06:45:06.324 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:06 smithi125 bash[80002]: cluster 2023-10-09T06:45:05.384559+0000 mgr.y (mgr.25723) 1215 : cluster 0 pgmap v1308: 73 pgs: 18 peering, 55 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 170 B/s rd, 0 op/s; 184 B/s, 0 objects/s recovering 2023-10-09T06:45:06.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:06.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:06.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:06.513 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:06.576 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:06.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:06.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:06.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:06.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:06.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:06.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:06.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:06.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:07.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:07.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:07.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:07.153 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_5' created 2023-10-09T06:45:07.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.119578+0000 mon.a (mon.0) 1513 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.130464+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.132408+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e287: 8 total, 8 up, 5 in 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.135035+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.135212+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: audit 2023-10-09T06:45:06.474584+0000 mon.a (mon.0) 1515 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.561437+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.257 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:07 smithi155 bash[59495]: cluster 2023-10-09T06:45:06.561504+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.274 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:07.378 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.119578+0000 mon.a (mon.0) 1513 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.130464+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.132408+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e287: 8 total, 8 up, 5 in 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.135035+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.135212+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: audit 2023-10-09T06:45:06.474584+0000 mon.a (mon.0) 1515 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.561437+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[77299]: cluster 2023-10-09T06:45:06.561504+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.119578+0000 mon.a (mon.0) 1513 : cluster 3 Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.130464+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.132408+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e287: 8 total, 8 up, 5 in 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.135035+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.135212+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.389 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: audit 2023-10-09T06:45:06.474584+0000 mon.a (mon.0) 1515 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:45:07.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.561437+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.390 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:07 smithi125 bash[80002]: cluster 2023-10-09T06:45:06.561504+0000 osd.4 (osd.4) 247 : cluster 3 failed to encode map e287 with expected crc 2023-10-09T06:45:07.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:07.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:07.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:07.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:07.667 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:07.770 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:07.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:07.905 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:07.940 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:07.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:08.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:08.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:08.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:08.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.140903+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: audit 2023-10-09T06:45:07.140960+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:45:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.141076+0000 mon.a (mon.0) 1517 : cluster 0 osdmap e288: 8 total, 8 up, 5 in 2023-10-09T06:45:08.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.143494+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.143858+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.147598+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.147659+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:08 smithi155 bash[59495]: cluster 2023-10-09T06:45:07.385020+0000 mgr.y (mgr.25723) 1216 : cluster 0 pgmap v1311: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 255 B/s rd, 0 op/s; 276 B/s, 0 objects/s recovering 2023-10-09T06:45:08.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.140903+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: audit 2023-10-09T06:45:07.140960+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:45:08.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.141076+0000 mon.a (mon.0) 1517 : cluster 0 osdmap e288: 8 total, 8 up, 5 in 2023-10-09T06:45:08.441 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.143494+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.143858+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.147598+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.147659+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[77299]: cluster 2023-10-09T06:45:07.385020+0000 mgr.y (mgr.25723) 1216 : cluster 0 pgmap v1311: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 255 B/s rd, 0 op/s; 276 B/s, 0 objects/s recovering 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.140903+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: audit 2023-10-09T06:45:07.140960+0000 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.125:0/2353013492' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.141076+0000 mon.a (mon.0) 1517 : cluster 0 osdmap e288: 8 total, 8 up, 5 in 2023-10-09T06:45:08.442 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.143494+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.143858+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.147598+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.147659+0000 osd.4 (osd.4) 248 : cluster 3 failed to encode map e288 with expected crc 2023-10-09T06:45:08.443 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:08 smithi125 bash[80002]: cluster 2023-10-09T06:45:07.385020+0000 mgr.y (mgr.25723) 1216 : cluster 0 pgmap v1311: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 255 B/s rd, 0 op/s; 276 B/s, 0 objects/s recovering 2023-10-09T06:45:08.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:08.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:08.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:08.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:08.702 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:08.729 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:08.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:08.835 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:08.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:08.962 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:08.991 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:08.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:09.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:09.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.261 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:09.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.148096+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.149288+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.149522+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e289: 8 total, 8 up, 5 in 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.151908+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.152124+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.153042+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.378 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:09 smithi155 bash[59495]: cluster 2023-10-09T06:45:08.156115+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:09.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.532 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:09.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.148096+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.149288+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.149522+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e289: 8 total, 8 up, 5 in 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.151908+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.152124+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.153042+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[77299]: cluster 2023-10-09T06:45:08.156115+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.148096+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.149288+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.149522+0000 mon.a (mon.0) 1518 : cluster 0 osdmap e289: 8 total, 8 up, 5 in 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.151908+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.152124+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.153042+0000 osd.4 (osd.4) 249 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:09 smithi125 bash[80002]: cluster 2023-10-09T06:45:08.156115+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e289 with expected crc 2023-10-09T06:45:09.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:09.686 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T06:45:09.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:09.905 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:09.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:09.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:09.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:10.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:10.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:10.168 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:10.180 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:10.189 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.153564+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e290: 8 total, 8 up, 5 in 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.155821+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.158788+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.385694+0000 mgr.y (mgr.25723) 1217 : cluster 0 pgmap v1314: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.470242+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.566023+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[80002]: cluster 2023-10-09T06:45:09.566092+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.153564+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e290: 8 total, 8 up, 5 in 2023-10-09T06:45:10.190 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.155821+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.158788+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.385694+0000 mgr.y (mgr.25723) 1217 : cluster 0 pgmap v1314: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:10.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.470242+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.566023+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.191 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:10 smithi125 bash[77299]: cluster 2023-10-09T06:45:09.566092+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:10.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:10.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:10.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:10.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.153564+0000 mon.a (mon.0) 1519 : cluster 0 osdmap e290: 8 total, 8 up, 5 in 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.155821+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.158788+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.385694+0000 mgr.y (mgr.25723) 1217 : cluster 0 pgmap v1314: 89 pgs: 16 unknown, 1 remapped+peering, 18 peering, 54 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.470242+0000 osd.4 (osd.4) 250 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.566023+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:10 smithi155 bash[59495]: cluster 2023-10-09T06:45:09.566092+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e290 with expected crc 2023-10-09T06:45:10.439 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:10.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:10.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:10.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:10.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:10.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:10.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:10.872 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:10.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:10.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:11.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:11.093 INFO:tasks.thrashosds.thrasher:in_osds: [0, 3, 4, 7, 6] out_osds: [5, 1, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:45:11.093 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:45:11.094 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 3, 4, 7, 6] 2023-10-09T06:45:11.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T06:45:11.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:11.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:11.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:11.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:11.273 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:11.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:11.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:11] "GET /metrics HTTP/1.1" 200 34176 "" "Prometheus/2.33.4" 2023-10-09T06:45:11.405 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:11.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:11.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:11.538 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:11.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:11.651 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:11 smithi125 bash[77299]: cluster 2023-10-09T06:45:11.438283+0000 mon.a (mon.0) 1520 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:45:11.651 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:11 smithi125 bash[80002]: cluster 2023-10-09T06:45:11.438283+0000 mon.a (mon.0) 1520 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:45:11.675 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:11.779 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:11.792 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:11 smithi155 bash[59495]: cluster 2023-10-09T06:45:11.438283+0000 mon.a (mon.0) 1520 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-10-09T06:45:11.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:11.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:11.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:11.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:12.050 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:12.078 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:12.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:12.195 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:12.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:12.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:12.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:12.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:12.474 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:12.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:12.589 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:12 smithi155 bash[59495]: cluster 2023-10-09T06:45:11.386993+0000 mgr.y (mgr.25723) 1218 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 779 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T06:45:12.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:12.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:12.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:12.719 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:12 smithi125 bash[77299]: cluster 2023-10-09T06:45:11.386993+0000 mgr.y (mgr.25723) 1218 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 779 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T06:45:12.720 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:12 smithi125 bash[80002]: cluster 2023-10-09T06:45:11.386993+0000 mgr.y (mgr.25723) 1218 : cluster 0 pgmap v1315: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 779 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T06:45:12.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:12.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:12.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:12.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:12.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:13.107 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:13.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:13.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:13.384 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:13.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:13.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:13.647 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:13.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:13.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:13.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:13.947 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:13.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:14.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:14.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:14.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:14.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:14.226 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:14.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:14.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:14.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:14.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:14.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:14.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:14.617 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:14 smithi125 bash[77299]: cluster 2023-10-09T06:45:13.387545+0000 mgr.y (mgr.25723) 1219 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 656 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T06:45:14.617 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:14 smithi125 bash[80002]: cluster 2023-10-09T06:45:13.387545+0000 mgr.y (mgr.25723) 1219 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 656 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T06:45:14.718 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:14.753 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:14.754 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:14 smithi155 bash[59495]: cluster 2023-10-09T06:45:13.387545+0000 mgr.y (mgr.25723) 1219 : cluster 0 pgmap v1316: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 656 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T06:45:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:14.856 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:14.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:14.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:14.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:14.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:15.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:15.114 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:15.144 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:15.239 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:15.249 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:15.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:15.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:15.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:15.522 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:15.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:15.657 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:15.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:15.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:15.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:15.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:15.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:15.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:15.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:16.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:16.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:16.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:16.313 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.335 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:16.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:16.459 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:16.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:16.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:16.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:16.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:16.719 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:16.721 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:16 smithi125 bash[77299]: cluster 2023-10-09T06:45:15.388365+0000 mgr.y (mgr.25723) 1220 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 512 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:16.721 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:16 smithi125 bash[80002]: cluster 2023-10-09T06:45:15.388365+0000 mgr.y (mgr.25723) 1220 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 512 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:16.823 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.846 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:16.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:16 smithi155 bash[59495]: cluster 2023-10-09T06:45:15.388365+0000 mgr.y (mgr.25723) 1220 : cluster 0 pgmap v1317: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 512 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:16.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.973 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:16.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:17.076 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:17.113 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:17.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:17.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:17.223 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:45:17.224 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:45:17.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:17.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:17.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:17.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:17.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:17.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:17.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:17.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:17.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:17.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:17.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:17.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:17.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:18.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:18.045 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:18.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:18.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:18.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:18.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:18.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.311 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:18.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:18.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:18.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:18.553 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:18 smithi125 bash[77299]: cluster 2023-10-09T06:45:17.389247+0000 mgr.y (mgr.25723) 1221 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 443 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:18.553 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:18 smithi125 bash[80002]: cluster 2023-10-09T06:45:17.389247+0000 mgr.y (mgr.25723) 1221 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 443 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:18.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:18.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:18.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.818 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:18.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:18 smithi155 bash[59495]: cluster 2023-10-09T06:45:17.389247+0000 mgr.y (mgr.25723) 1221 : cluster 0 pgmap v1318: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 443 KiB/s, 1 keys/s, 5 objects/s recovering 2023-10-09T06:45:18.921 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.945 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:18.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:19.048 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:19.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:19.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:19.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:19.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:19.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.460 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:19.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:19 smithi125 bash[80002]: audit 2023-10-09T06:45:18.606537+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:19.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:19 smithi125 bash[80002]: audit 2023-10-09T06:45:19.324566+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:19.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:19 smithi125 bash[77299]: audit 2023-10-09T06:45:18.606537+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:19.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:19 smithi125 bash[77299]: audit 2023-10-09T06:45:19.324566+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:19 smithi155 bash[59495]: audit 2023-10-09T06:45:18.606537+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:19 smithi155 bash[59495]: audit 2023-10-09T06:45:19.324566+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:19.697 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:19.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:19.826 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:19.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:19.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:19.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:20.086 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:20.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:20.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:20.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:20.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.514 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:20.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:20.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:20.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:20.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:20.698 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:20 smithi125 bash[77299]: cluster 2023-10-09T06:45:19.389782+0000 mgr.y (mgr.25723) 1222 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 400 KiB/s, 0 keys/s, 4 objects/s recovering 2023-10-09T06:45:20.698 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:20 smithi125 bash[80002]: cluster 2023-10-09T06:45:19.389782+0000 mgr.y (mgr.25723) 1222 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 400 KiB/s, 0 keys/s, 4 objects/s recovering 2023-10-09T06:45:20.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:20.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:20.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:20.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:20.884 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:20 smithi155 bash[59495]: cluster 2023-10-09T06:45:19.389782+0000 mgr.y (mgr.25723) 1222 : cluster 0 pgmap v1319: 89 pgs: 89 active+clean; 4.6 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 400 KiB/s, 0 keys/s, 4 objects/s recovering 2023-10-09T06:45:20.898 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:20.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:21.002 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:21.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:21.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:21.159 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:21.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:21.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:21.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:21.389 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:21] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-10-09T06:45:21.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:21.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:21.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:21.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:21.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:21.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:21 smithi125 bash[77299]: cluster 2023-10-09T06:45:21.390595+0000 mgr.y (mgr.25723) 1223 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 341 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:45:21.667 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:21 smithi125 bash[80002]: cluster 2023-10-09T06:45:21.390595+0000 mgr.y (mgr.25723) 1223 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 341 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:45:21.681 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:21.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:21.805 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:21 smithi155 bash[59495]: cluster 2023-10-09T06:45:21.390595+0000 mgr.y (mgr.25723) 1223 : cluster 0 pgmap v1320: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 341 KiB/s, 0 keys/s, 3 objects/s recovering 2023-10-09T06:45:21.908 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:21.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:21.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:21.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:45:22.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:22.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:22.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:22.309 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:22.345 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:22.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:22.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:22.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:22.702 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:22.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:22.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.860 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:22.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:22.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:22.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:22.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:23.089 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:23.217 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.247 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:23.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:23.485 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:23.618 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:23.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:23.754 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:23.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:23.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:23.917 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:23.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:24.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:24.050 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:24.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:24.186 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:24.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:24.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:24.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:24.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:24.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:24.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.714 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.718 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:24 smithi155 bash[59495]: cluster 2023-10-09T06:45:23.391187+0000 mgr.y (mgr.25723) 1224 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:24.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:24.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:24 smithi125 bash[77299]: cluster 2023-10-09T06:45:23.391187+0000 mgr.y (mgr.25723) 1224 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:24.827 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:24 smithi125 bash[80002]: cluster 2023-10-09T06:45:23.391187+0000 mgr.y (mgr.25723) 1224 : cluster 0 pgmap v1321: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:24.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:24.943 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:24.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:25.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:25.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:25.220 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:25.241 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:25.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:25.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:25.472 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:45:25.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:25.491 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:45:25.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:25.504 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:25 smithi125 bash[77299]: audit 2023-10-09T06:45:25.446393+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.125:0/1596056592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:25.512 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:45:25.537 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:45:25.558 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:45:25.587 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:45:25.602 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:25.610 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:45:25.615 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:25 smithi155 bash[59495]: audit 2023-10-09T06:45:25.446393+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.125:0/1596056592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:25.633 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:45:25.634 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.634 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.634 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.635 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.635 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.635 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.635 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.635 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:45:25.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:25.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:25.772 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:25 smithi125 bash[80002]: audit 2023-10-09T06:45:25.446393+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.125:0/1596056592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:25.774 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:25.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:25.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:25.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:25.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:25.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:26.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:26.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:26.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:26.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:26.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:26.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:26 smithi155 bash[59495]: cluster 2023-10-09T06:45:25.391783+0000 mgr.y (mgr.25723) 1225 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:26.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:26.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:26.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:26.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:26.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:26.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:26.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:26 smithi125 bash[77299]: cluster 2023-10-09T06:45:25.391783+0000 mgr.y (mgr.25723) 1225 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:26.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:26 smithi125 bash[80002]: cluster 2023-10-09T06:45:25.391783+0000 mgr.y (mgr.25723) 1225 : cluster 0 pgmap v1322: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:26.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:26.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:26.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:27.054 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:27.083 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:27.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:27.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:27.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:27.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:27.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:27.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:27.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:27.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:27.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:27.732 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:27.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:27.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:27.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:27.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:27.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:28.103 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:28.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:28.229 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:28.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:28.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:28.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:28.475 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:45:28.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:28.504 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: cluster 2023-10-09T06:45:27.392476+0000 mgr.y (mgr.25723) 1226 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:28.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: audit 2023-10-09T06:45:27.630707+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.125:0/2793572598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: audit 2023-10-09T06:45:27.631305+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: audit 2023-10-09T06:45:27.967710+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:28.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: audit 2023-10-09T06:45:28.342044+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.505 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:28 smithi155 bash[59495]: audit 2023-10-09T06:45:28.348163+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.521 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:28.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:28.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:28.750 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:28.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: cluster 2023-10-09T06:45:27.392476+0000 mgr.y (mgr.25723) 1226 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:28.757 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: audit 2023-10-09T06:45:27.630707+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.125:0/2793572598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: audit 2023-10-09T06:45:27.631305+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: audit 2023-10-09T06:45:27.967710+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: audit 2023-10-09T06:45:28.342044+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[77299]: audit 2023-10-09T06:45:28.348163+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: cluster 2023-10-09T06:45:27.392476+0000 mgr.y (mgr.25723) 1226 : cluster 0 pgmap v1323: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: audit 2023-10-09T06:45:27.630707+0000 mon.c (mon.2) 116 : audit 1 from='client.? 172.21.15.125:0/2793572598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: audit 2023-10-09T06:45:27.631305+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:45:28.758 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: audit 2023-10-09T06:45:27.967710+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:28.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: audit 2023-10-09T06:45:28.342044+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.759 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:28 smithi125 bash[80002]: audit 2023-10-09T06:45:28.348163+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:28.772 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:28.875 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:28.906 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:28.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:28.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:29.010 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:29.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:29.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:29.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:29.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:29.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.442 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:29.546 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:29.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.459928+0000 mon.a (mon.0) 1527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: audit 2023-10-09T06:45:28.464881+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.464948+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e291: 8 total, 8 up, 5 in 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.465087+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.745850+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.918677+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[77299]: cluster 2023-10-09T06:45:28.921789+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.459928+0000 mon.a (mon.0) 1527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: audit 2023-10-09T06:45:28.464881+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.464948+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e291: 8 total, 8 up, 5 in 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.465087+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.745850+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.918677+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:29 smithi125 bash[80002]: cluster 2023-10-09T06:45:28.921789+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.459928+0000 mon.a (mon.0) 1527 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: audit 2023-10-09T06:45:28.464881+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.464948+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e291: 8 total, 8 up, 5 in 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.465087+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.745850+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.918677+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:29 smithi155 bash[59495]: cluster 2023-10-09T06:45:28.921789+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:29.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:29.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:29.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:29.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:29.827 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:29.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:29.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:29.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:29.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:30.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:30.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.225 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:30.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:30.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:30.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:30.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:30 smithi155 bash[59495]: cluster 2023-10-09T06:45:29.099264+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:30.613 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:30 smithi155 bash[59495]: cluster 2023-10-09T06:45:29.393105+0000 mgr.y (mgr.25723) 1227 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:30.630 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:30.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.765 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:30.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:30 smithi125 bash[80002]: cluster 2023-10-09T06:45:29.099264+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:30.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:30 smithi125 bash[80002]: cluster 2023-10-09T06:45:29.393105+0000 mgr.y (mgr.25723) 1227 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:30.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:30 smithi125 bash[77299]: cluster 2023-10-09T06:45:29.099264+0000 osd.4 (osd.4) 251 : cluster 3 failed to encode map e291 with expected crc 2023-10-09T06:45:30.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:30 smithi125 bash[77299]: cluster 2023-10-09T06:45:29.393105+0000 mgr.y (mgr.25723) 1227 : cluster 0 pgmap v1325: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:30.903 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:30.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:31.007 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:31.036 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:31.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:31.166 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:31.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:31.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:31.350 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:45:31.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:31.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:31.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:31.557 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:31] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.33.4" 2023-10-09T06:45:31.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:31.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:31.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:31.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:31.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:31.932 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:31.964 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:31.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:32.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:32.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.240 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:32.343 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:32.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:32.480 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: debug 2023-10-09T06:45:32.230+0000 7ff949fd9700 -1 mon.a@0(leader).osd e291 definitely_dead 0 2023-10-09T06:45:32.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: cluster 2023-10-09T06:45:31.393968+0000 mgr.y (mgr.25723) 1228 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:32.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: audit 2023-10-09T06:45:32.092012+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.125:0/93339829' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: audit 2023-10-09T06:45:32.092743+0000 mon.a (mon.0) 1530 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: audit 2023-10-09T06:45:32.234231+0000 mon.a (mon.0) 1531 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:45:32.502 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[77299]: cluster 2023-10-09T06:45:32.234402+0000 mon.a (mon.0) 1532 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:45:32.508 INFO:teuthology.orchestra.run.smithi125.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T06:45:32.509 INFO:teuthology.orchestra.run.smithi125.stderr:marked out osd.3. 2023-10-09T06:45:32.519 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:32.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:32.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:32.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:32.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[80002]: cluster 2023-10-09T06:45:31.393968+0000 mgr.y (mgr.25723) 1228 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:32.770 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[80002]: audit 2023-10-09T06:45:32.092012+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.125:0/93339829' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[80002]: audit 2023-10-09T06:45:32.092743+0000 mon.a (mon.0) 1530 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[80002]: audit 2023-10-09T06:45:32.234231+0000 mon.a (mon.0) 1531 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:45:32.771 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:32 smithi125 bash[80002]: cluster 2023-10-09T06:45:32.234402+0000 mon.a (mon.0) 1532 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:45:32.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:32.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:32 smithi155 bash[59495]: cluster 2023-10-09T06:45:31.393968+0000 mgr.y (mgr.25723) 1228 : cluster 0 pgmap v1326: 89 pgs: 89 active+clean; 4.6 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail 2023-10-09T06:45:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:32 smithi155 bash[59495]: audit 2023-10-09T06:45:32.092012+0000 mon.c (mon.2) 117 : audit 1 from='client.? 172.21.15.125:0/93339829' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:32 smithi155 bash[59495]: audit 2023-10-09T06:45:32.092743+0000 mon.a (mon.0) 1530 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:45:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:32 smithi155 bash[59495]: audit 2023-10-09T06:45:32.234231+0000 mon.a (mon.0) 1531 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:45:32.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:32 smithi155 bash[59495]: cluster 2023-10-09T06:45:32.234402+0000 mon.a (mon.0) 1532 : cluster 1 Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:45:32.926 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:32.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:33.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:33.060 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:33.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:33.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:33.297 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:33.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:33.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:45:33.428 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:33.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:33.556 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:33.580 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:33.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: cluster 2023-10-09T06:45:32.488863+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: audit 2023-10-09T06:45:32.491861+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: audit 2023-10-09T06:45:32.491985+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: cluster 2023-10-09T06:45:32.492095+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e292: 8 total, 8 up, 4 in 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: cluster 2023-10-09T06:45:32.492547+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: cluster 2023-10-09T06:45:32.492649+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.489625+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e293: 8 total, 8 up, 4 in 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: cluster 2023-10-09T06:45:32.488863+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: audit 2023-10-09T06:45:32.491861+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: audit 2023-10-09T06:45:32.491985+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: cluster 2023-10-09T06:45:32.492095+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e292: 8 total, 8 up, 4 in 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: cluster 2023-10-09T06:45:32.492547+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: cluster 2023-10-09T06:45:32.492649+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:33 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.489625+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e293: 8 total, 8 up, 4 in 2023-10-09T06:45:33.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:33.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:33.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:33.817 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: cluster 2023-10-09T06:45:32.488863+0000 osd.4 (osd.4) 252 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: audit 2023-10-09T06:45:32.491861+0000 mon.a (mon.0) 1533 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: audit 2023-10-09T06:45:32.491985+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.125:0/3894552135' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: cluster 2023-10-09T06:45:32.492095+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e292: 8 total, 8 up, 4 in 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: cluster 2023-10-09T06:45:32.492547+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: cluster 2023-10-09T06:45:32.492649+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e292 with expected crc 2023-10-09T06:45:33.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:33 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.489625+0000 mon.a (mon.0) 1536 : cluster 0 osdmap e293: 8 total, 8 up, 4 in 2023-10-09T06:45:33.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:33.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:33.960 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:33.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:34.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:34.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:34.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:34.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:34.311 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:34.335 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:34.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:34.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:34.614 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.394529+0000 mgr.y (mgr.25723) 1229 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.492298+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.492413+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.492786+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.496465+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.497801+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.498000+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:33.500700+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: audit 2023-10-09T06:45:33.606400+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:34.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:34 smithi155 bash[59495]: cluster 2023-10-09T06:45:34.492399+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e294: 8 total, 8 up, 4 in 2023-10-09T06:45:34.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.394529+0000 mgr.y (mgr.25723) 1229 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail 2023-10-09T06:45:34.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.492298+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.492413+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.492786+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.727 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.496465+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.497801+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.498000+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:33.500700+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: audit 2023-10-09T06:45:33.606400+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[77299]: cluster 2023-10-09T06:45:34.492399+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e294: 8 total, 8 up, 4 in 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.394529+0000 mgr.y (mgr.25723) 1229 : cluster 0 pgmap v1328: 89 pgs: 89 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.492298+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.492413+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.492786+0000 osd.4 (osd.4) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.496465+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.497801+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.498000+0000 osd.4 (osd.4) 254 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:33.500700+0000 osd.4 (osd.4) 255 : cluster 3 failed to encode map e293 with expected crc 2023-10-09T06:45:34.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: audit 2023-10-09T06:45:33.606400+0000 mon.a (mon.0) 1537 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:34.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:34 smithi125 bash[80002]: cluster 2023-10-09T06:45:34.492399+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e294: 8 total, 8 up, 4 in 2023-10-09T06:45:34.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:34.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:34.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:34.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:35.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:35.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:35.042 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:35.106 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:35.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:35.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:35.259 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:35.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:35.392 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:35.495 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:35.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:35.535 DEBUG:teuthology.orchestra.run.smithi125:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T06:45:35.575 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:45:35.575 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:45:35.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:35.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:35.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:35.767 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: cluster 2023-10-09T06:45:34.492262+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: cluster 2023-10-09T06:45:34.494921+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: cluster 2023-10-09T06:45:34.494924+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: cluster 2023-10-09T06:45:34.496594+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: audit 2023-10-09T06:45:34.649538+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: audit 2023-10-09T06:45:34.656950+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.395818+0000 mgr.y (mgr.25723) 1230 : cluster 0 pgmap v1331: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 39 B/s, 8 objects/s recovering 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: cluster 2023-10-09T06:45:34.492262+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: cluster 2023-10-09T06:45:34.494921+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.768 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: cluster 2023-10-09T06:45:34.494924+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: cluster 2023-10-09T06:45:34.496594+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:35.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: audit 2023-10-09T06:45:34.649538+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:35.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: audit 2023-10-09T06:45:34.656950+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:35.769 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:35 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.395818+0000 mgr.y (mgr.25723) 1230 : cluster 0 pgmap v1331: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 39 B/s, 8 objects/s recovering 2023-10-09T06:45:35.782 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:35.886 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:35.914 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:35.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:36.018 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:36.040 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:36.135 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:hints = 1 2023-10-09T06:45:36.136 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:45:36.136 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object prefix: benchmark_data_smithi125_222801 2023-10-09T06:45:36.136 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:45:36.136 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:45:36.144 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: cluster 2023-10-09T06:45:34.492262+0000 osd.4 (osd.4) 256 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: cluster 2023-10-09T06:45:34.494921+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: cluster 2023-10-09T06:45:34.494924+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: cluster 2023-10-09T06:45:34.496594+0000 osd.4 (osd.4) 257 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: audit 2023-10-09T06:45:34.649538+0000 mon.a (mon.0) 1539 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:36.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: audit 2023-10-09T06:45:34.656950+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:36.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:35 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.395818+0000 mgr.y (mgr.25723) 1230 : cluster 0 pgmap v1331: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 39 B/s, 8 objects/s recovering 2023-10-09T06:45:36.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:36.278 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:36.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:36.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:36.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:36.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:36.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:36.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:36.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.269545+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.691 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.654589+0000 mon.a (mon.0) 1541 : cluster 3 Health check failed: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:45:36.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.667236+0000 mon.a (mon.0) 1542 : cluster 0 osdmap e295: 8 total, 8 up, 4 in 2023-10-09T06:45:36.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.669836+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.673023+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.692 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:36 smithi155 bash[59495]: cluster 2023-10-09T06:45:35.675550+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:36.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.269545+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.654589+0000 mon.a (mon.0) 1541 : cluster 3 Health check failed: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:45:36.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.667236+0000 mon.a (mon.0) 1542 : cluster 0 osdmap e295: 8 total, 8 up, 4 in 2023-10-09T06:45:36.791 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.669836+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.673023+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[77299]: cluster 2023-10-09T06:45:35.675550+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.269545+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e294 with expected crc 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.654589+0000 mon.a (mon.0) 1541 : cluster 3 Health check failed: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.667236+0000 mon.a (mon.0) 1542 : cluster 0 osdmap e295: 8 total, 8 up, 4 in 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.669836+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.792 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.673023+0000 osd.4 (osd.4) 258 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.793 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:36 smithi125 bash[80002]: cluster 2023-10-09T06:45:35.675550+0000 osd.4 (osd.4) 259 : cluster 3 failed to encode map e295 with expected crc 2023-10-09T06:45:36.814 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:36.946 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:36.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:36.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:37.079 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:37.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:37.116 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:37.136 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 1 16 2452 2436 152.241 152.25 0.00511402 0.00654324 2023-10-09T06:45:37.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:37.358 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:37.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:37.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:37.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:37.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:37.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:36.668731+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e296: 8 total, 8 up, 4 in 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:36.671221+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:36.671642+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:36.672424+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:36.745374+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[77299]: cluster 2023-10-09T06:45:37.396392+0000 mgr.y (mgr.25723) 1231 : cluster 0 pgmap v1334: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 58 B/s, 12 objects/s recovering 2023-10-09T06:45:37.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:36.668731+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e296: 8 total, 8 up, 4 in 2023-10-09T06:45:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:36.671221+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:36.671642+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:36.672424+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:36.745374+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:37 smithi125 bash[80002]: cluster 2023-10-09T06:45:37.396392+0000 mgr.y (mgr.25723) 1231 : cluster 0 pgmap v1334: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 58 B/s, 12 objects/s recovering 2023-10-09T06:45:37.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:36.668731+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e296: 8 total, 8 up, 4 in 2023-10-09T06:45:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:36.671221+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:36.671642+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:36.672424+0000 osd.4 (osd.4) 260 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:36.745374+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e296 with expected crc 2023-10-09T06:45:37.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:37 smithi155 bash[59495]: cluster 2023-10-09T06:45:37.396392+0000 mgr.y (mgr.25723) 1231 : cluster 0 pgmap v1334: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 58 B/s, 12 objects/s recovering 2023-10-09T06:45:37.924 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:37.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:38.028 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:38.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:38.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 2 16 4044 4028 125.865 99.5 0.00697102 0.00792332 2023-10-09T06:45:38.156 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:38.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:38.285 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:38.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:38.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:38.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:38.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:38.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:38.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:38.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:38.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:38.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:38.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:38.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:38.972 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:38.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:38 smithi155 bash[59495]: cluster 2023-10-09T06:45:37.676706+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e297: 8 total, 8 up, 4 in 2023-10-09T06:45:38.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:38 smithi155 bash[59495]: cluster 2023-10-09T06:45:37.678360+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:38.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:38 smithi155 bash[59495]: cluster 2023-10-09T06:45:37.681198+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:38.983 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:38 smithi155 bash[59495]: cluster 2023-10-09T06:45:37.683481+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:39.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.135 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:39.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 3 16 6329 6313 131.51 142.812 0.00762306 0.00758571 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[77299]: cluster 2023-10-09T06:45:37.676706+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e297: 8 total, 8 up, 4 in 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[77299]: cluster 2023-10-09T06:45:37.678360+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[77299]: cluster 2023-10-09T06:45:37.681198+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[77299]: cluster 2023-10-09T06:45:37.683481+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[80002]: cluster 2023-10-09T06:45:37.676706+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e297: 8 total, 8 up, 4 in 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[80002]: cluster 2023-10-09T06:45:37.678360+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[80002]: cluster 2023-10-09T06:45:37.681198+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:38 smithi125 bash[80002]: cluster 2023-10-09T06:45:37.683481+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e297 with expected crc 2023-10-09T06:45:39.230 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:39.238 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.267 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:39.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:39.513 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:39.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:39.553 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 7, 6] out_osds: [5, 1, 2, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:45:39.553 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:45:39.553 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T06:45:39.553 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:45:39.553 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:45:39.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T06:45:39.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.668 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:39.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.809 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[77299]: audit 2023-10-09T06:45:38.717553+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[77299]: audit 2023-10-09T06:45:38.724688+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[77299]: cluster 2023-10-09T06:45:39.396930+0000 mgr.y (mgr.25723) 1232 : cluster 0 pgmap v1336: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.1 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 48 B/s, 10 objects/s recovering 2023-10-09T06:45:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[80002]: audit 2023-10-09T06:45:38.717553+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[80002]: audit 2023-10-09T06:45:38.724688+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:39 smithi125 bash[80002]: cluster 2023-10-09T06:45:39.396930+0000 mgr.y (mgr.25723) 1232 : cluster 0 pgmap v1336: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.1 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 48 B/s, 10 objects/s recovering 2023-10-09T06:45:39.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.923 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:39 smithi155 bash[59495]: audit 2023-10-09T06:45:38.717553+0000 mon.a (mon.0) 1545 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.923 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:39 smithi155 bash[59495]: audit 2023-10-09T06:45:38.724688+0000 mon.a (mon.0) 1546 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:39.924 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:39 smithi155 bash[59495]: cluster 2023-10-09T06:45:39.396930+0000 mgr.y (mgr.25723) 1232 : cluster 0 pgmap v1336: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 84 active+clean; 4.6 MiB data, 3.1 GiB used, 355 GiB / 358 GiB avail; 43/84 objects degraded (51.190%); 48 B/s, 10 objects/s recovering 2023-10-09T06:45:39.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:39.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:40.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:40.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:40.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 4 16 8511 8495 132.723 136.375 0.00673977 0.0075209 2023-10-09T06:45:40.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:40.211 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:40.314 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:40.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:40.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:40.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:40.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:40.601 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:40.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:40.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:40.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:40.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:40.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:40.966 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:40.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:40.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:41.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:41.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:41.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 5 16 10509 10493 131.152 124.875 0.0679706 0.00754869 2023-10-09T06:45:41.227 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:41.257 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:41.363 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:41.391 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:41.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:41.527 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:41.628 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:41] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.33.4" 2023-10-09T06:45:41.631 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:41.657 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:41.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:41.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:41 smithi155 bash[59495]: audit 2023-10-09T06:45:40.621437+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.769 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:41 smithi155 bash[59495]: audit 2023-10-09T06:45:40.629480+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.770 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:41 smithi155 bash[59495]: cluster 2023-10-09T06:45:41.397792+0000 mgr.y (mgr.25723) 1233 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 459 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s; 0 B/s, 7 objects/s recovering 2023-10-09T06:45:41.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:41.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[80002]: audit 2023-10-09T06:45:40.621437+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[80002]: audit 2023-10-09T06:45:40.629480+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[80002]: cluster 2023-10-09T06:45:41.397792+0000 mgr.y (mgr.25723) 1233 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 459 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s; 0 B/s, 7 objects/s recovering 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[77299]: audit 2023-10-09T06:45:40.621437+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[77299]: audit 2023-10-09T06:45:40.629480+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:41.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:41 smithi125 bash[77299]: cluster 2023-10-09T06:45:41.397792+0000 mgr.y (mgr.25723) 1233 : cluster 0 pgmap v1337: 89 pgs: 89 active+clean; 459 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s; 0 B/s, 7 objects/s recovering 2023-10-09T06:45:41.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:41.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:41.920 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:41.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:42.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:42.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:42.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 6 16 12319 12303 128.145 113.125 0.0041128 0.00779235 2023-10-09T06:45:42.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:42.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:42.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:42.317 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:42.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:42.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:42.581 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:42.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:42.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:42 smithi155 bash[59495]: cluster 2023-10-09T06:45:41.628725+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded) 2023-10-09T06:45:42.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.694 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:42 smithi125 bash[77299]: cluster 2023-10-09T06:45:41.628725+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded) 2023-10-09T06:45:42.694 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:42 smithi125 bash[80002]: cluster 2023-10-09T06:45:41.628725+0000 mon.a (mon.0) 1549 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/84 objects degraded (51.190%), 4 pgs degraded) 2023-10-09T06:45:42.710 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:42.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:42.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:42.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:42.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:42.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:43.071 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:43.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:43.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 7 16 14194 14178 126.578 117.188 0.0106822 0.00789216 2023-10-09T06:45:43.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:43.231 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:43.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:43.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:43.467 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:43.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:43.593 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:43.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:43.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:43.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:43 smithi155 bash[59495]: cluster 2023-10-09T06:45:43.398474+0000 mgr.y (mgr.25723) 1234 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 459 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 59 MiB/s wr, 940 op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:43.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:43.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:43.790 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:43 smithi125 bash[77299]: cluster 2023-10-09T06:45:43.398474+0000 mgr.y (mgr.25723) 1234 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 459 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 59 MiB/s wr, 940 op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:43.790 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:43 smithi125 bash[80002]: cluster 2023-10-09T06:45:43.398474+0000 mgr.y (mgr.25723) 1234 : cluster 0 pgmap v1338: 89 pgs: 89 active+clean; 459 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 59 MiB/s wr, 940 op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:43.859 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:43.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:43.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:43.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:43.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:44.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:44.131 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:44.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 8 16 15918 15902 124.224 107.75 0.0062315 0.00803985 2023-10-09T06:45:44.156 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:44.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.286 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:44.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:44.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:44.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.671 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:44.775 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:44.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:44.901 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:44.929 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:44.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:45.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:45.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:45.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 9 16 17816 17800 123.6 118.625 0.01174 0.00808293 2023-10-09T06:45:45.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:45.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:45.268 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:45:45.269 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:45:45.269 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T06:45:45.269 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:45:45.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:45.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:45.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:45.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:45.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:45.578 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:45.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:45.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:45.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:45.851 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:45.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:45.955 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:45.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:45.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:45.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:46.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:46.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 10 16 19411 19395 121.208 99.6875 0.0132183 0.00824467 2023-10-09T06:45:46.214 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:46.240 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:46.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:46.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.369 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:46.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:46.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:46.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:46.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:46.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:46 smithi125 bash[77299]: cluster 2023-10-09T06:45:45.399348+0000 mgr.y (mgr.25723) 1235 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 120 MiB/s wr, 1.92k op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:46.609 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:46 smithi125 bash[80002]: cluster 2023-10-09T06:45:45.399348+0000 mgr.y (mgr.25723) 1235 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 120 MiB/s wr, 1.92k op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:46.623 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:46.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:46.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:46.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:46.853 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.881 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:46.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:46.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:46 smithi155 bash[59495]: cluster 2023-10-09T06:45:45.399348+0000 mgr.y (mgr.25723) 1235 : cluster 0 pgmap v1339: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 120 MiB/s wr, 1.92k op/s; 0 B/s, 5 objects/s recovering 2023-10-09T06:45:46.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:47.015 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:47.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:47.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 11 16 21163 21147 120.143 109.5 0.00576497 0.00831419 2023-10-09T06:45:47.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:47.260 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:47.289 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:47.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:47.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:47.520 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:47.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:47.656 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:47.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:47.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:47.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:47.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:47.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:47.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:47.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:48.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:48.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:48.137 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 12 16 22790 22774 118.604 101.688 0.0099893 0.00842299 2023-10-09T06:45:48.139 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:45:48.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:48.227 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:48.330 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:48.362 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:48.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:48.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:48.575 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:48.594 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:48.620 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:48 smithi155 bash[59495]: cluster 2023-10-09T06:45:47.399854+0000 mgr.y (mgr.25723) 1236 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 4 objects/s recovering 2023-10-09T06:45:48.723 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:48.734 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:48 smithi125 bash[77299]: cluster 2023-10-09T06:45:47.399854+0000 mgr.y (mgr.25723) 1236 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 4 objects/s recovering 2023-10-09T06:45:48.734 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:48 smithi125 bash[80002]: cluster 2023-10-09T06:45:47.399854+0000 mgr.y (mgr.25723) 1236 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s; 0 B/s, 4 objects/s recovering 2023-10-09T06:45:48.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:48.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:48.876 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:48.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:48.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:49.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:49.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 13 16 24165 24149 116.09 85.9375 0.00912941 0.00860747 2023-10-09T06:45:49.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:49.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:49.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:49.370 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:49.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:49.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:49.632 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:49.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:49 smithi125 bash[77299]: audit 2023-10-09T06:45:48.606780+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:49.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:49 smithi125 bash[77299]: audit 2023-10-09T06:45:49.343364+0000 mon.a (mon.0) 1551 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:49 smithi125 bash[80002]: audit 2023-10-09T06:45:48.606780+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:49.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:49 smithi125 bash[80002]: audit 2023-10-09T06:45:49.343364+0000 mon.a (mon.0) 1551 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:49.659 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:49.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:49.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:49.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:49 smithi155 bash[59495]: audit 2023-10-09T06:45:48.606780+0000 mon.a (mon.0) 1550 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:49 smithi155 bash[59495]: audit 2023-10-09T06:45:49.343364+0000 mon.a (mon.0) 1551 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:45:49.898 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:45:49.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:49.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:49.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:49.923 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:45:49.949 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:45:49.977 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:45:50.005 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:45:50.020 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:50.039 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:45:50.042 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:50.066 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:45:50.093 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.095 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:45:50.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 14 16 25741 25725 114.833 98.5 0.0103161 0.00870198 2023-10-09T06:45:50.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:50.168 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:50.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:50.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:50.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:50.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:50.434 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:50.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:50.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:50 smithi155 bash[59495]: cluster 2023-10-09T06:45:49.400272+0000 mgr.y (mgr.25723) 1237 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:50 smithi155 bash[59495]: audit 2023-10-09T06:45:49.862659+0000 mon.a (mon.0) 1552 : audit 0 from='client.? 172.21.15.125:0/3138753244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:50.665 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:50.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:50.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:50.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:50 smithi125 bash[77299]: cluster 2023-10-09T06:45:49.400272+0000 mgr.y (mgr.25723) 1237 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:50.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:50 smithi125 bash[77299]: audit 2023-10-09T06:45:49.862659+0000 mon.a (mon.0) 1552 : audit 0 from='client.? 172.21.15.125:0/3138753244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:50.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:50 smithi125 bash[80002]: cluster 2023-10-09T06:45:49.400272+0000 mgr.y (mgr.25723) 1237 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:50.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:50 smithi125 bash[80002]: audit 2023-10-09T06:45:49.862659+0000 mon.a (mon.0) 1552 : audit 0 from='client.? 172.21.15.125:0/3138753244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:45:50.920 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:50.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:50.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:51.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:51.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:51.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 15 16 27235 27219 113.402 93.375 0.00766695 0.00881102 2023-10-09T06:45:51.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:51.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:51.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:51.325 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:51.360 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:51.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:51.472 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:45:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:45:51] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T06:45:51.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:51.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:51.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:51.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:51.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:51.838 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:51.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:51.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:51.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:51.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:52.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:52.121 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:52.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 16 16 28716 28700 112.099 92.5625 0.0061201 0.00891427 2023-10-09T06:45:52.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:52.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:52.359 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:52.389 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:52.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:52.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:52 smithi155 bash[59495]: cluster 2023-10-09T06:45:51.400912+0000 mgr.y (mgr.25723) 1238 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 129 MiB/s wr, 2.07k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:52.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:52.623 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:52.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:52.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:52.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:52.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:52.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:52 smithi125 bash[77299]: cluster 2023-10-09T06:45:51.400912+0000 mgr.y (mgr.25723) 1238 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 129 MiB/s wr, 2.07k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:52.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:52 smithi125 bash[80002]: cluster 2023-10-09T06:45:51.400912+0000 mgr.y (mgr.25723) 1238 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 1.5 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 129 MiB/s wr, 2.07k op/s; 0 B/s, 3 objects/s recovering 2023-10-09T06:45:52.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:52.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:52.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:53.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:53.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 17 16 30001 29985 110.229 80.3125 0.00472112 0.00906482 2023-10-09T06:45:53.150 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.174 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:53.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:53.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:53.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.429 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:53.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:53.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:53.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:53.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:53.789 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:53.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:53.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:53.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:53.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:54.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:54.092 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:54.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 18 16 31433 31417 109.077 89.5 0.00856391 0.00916143 2023-10-09T06:45:54.196 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:54.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:54.297 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:54.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:54.352 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:54.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:54.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:54.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:54.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[80002]: cluster 2023-10-09T06:45:53.401424+0000 mgr.y (mgr.25723) 1239 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:45:54.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[80002]: audit 2023-10-09T06:45:54.354798+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.125:0/2898325945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[80002]: audit 2023-10-09T06:45:54.355221+0000 mon.a (mon.0) 1553 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[77299]: cluster 2023-10-09T06:45:53.401424+0000 mgr.y (mgr.25723) 1239 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:45:54.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[77299]: audit 2023-10-09T06:45:54.354798+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.125:0/2898325945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.597 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:54 smithi125 bash[77299]: audit 2023-10-09T06:45:54.355221+0000 mon.a (mon.0) 1553 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:54.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:54.732 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:54 smithi155 bash[59495]: cluster 2023-10-09T06:45:53.401424+0000 mgr.y (mgr.25723) 1239 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:45:54.732 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:54 smithi155 bash[59495]: audit 2023-10-09T06:45:54.354798+0000 mon.c (mon.2) 118 : audit 1 from='client.? 172.21.15.125:0/2898325945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.733 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:54 smithi155 bash[59495]: audit 2023-10-09T06:45:54.355221+0000 mon.a (mon.0) 1553 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:45:54.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:54.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:54.892 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:54.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:54.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:55.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:55.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:55.113 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:45:55.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:55.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 19 16 32734 32718 107.615 81.3125 0.0102101 0.00928664 2023-10-09T06:45:55.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:55.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:55.299 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:55.402 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.438 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:55.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:55.568 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:55 smithi155 bash[59495]: cluster 2023-10-09T06:45:55.096953+0000 mon.a (mon.0) 1554 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:45:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:55 smithi155 bash[59495]: cluster 2023-10-09T06:45:55.096991+0000 mon.a (mon.0) 1555 : cluster 1 Cluster is now healthy 2023-10-09T06:45:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:55 smithi155 bash[59495]: audit 2023-10-09T06:45:55.104515+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:55.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:55 smithi155 bash[59495]: cluster 2023-10-09T06:45:55.104696+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e298: 8 total, 8 up, 4 in 2023-10-09T06:45:55.671 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:55.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:45:55.840 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[80002]: cluster 2023-10-09T06:45:55.096953+0000 mon.a (mon.0) 1554 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:45:55.840 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[80002]: cluster 2023-10-09T06:45:55.096991+0000 mon.a (mon.0) 1555 : cluster 1 Cluster is now healthy 2023-10-09T06:45:55.840 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[80002]: audit 2023-10-09T06:45:55.104515+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:55.840 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[80002]: cluster 2023-10-09T06:45:55.104696+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e298: 8 total, 8 up, 4 in 2023-10-09T06:45:55.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[77299]: cluster 2023-10-09T06:45:55.096953+0000 mon.a (mon.0) 1554 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:45:55.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[77299]: cluster 2023-10-09T06:45:55.096991+0000 mon.a (mon.0) 1555 : cluster 1 Cluster is now healthy 2023-10-09T06:45:55.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[77299]: audit 2023-10-09T06:45:55.104515+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:45:55.841 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:55 smithi125 bash[77299]: cluster 2023-10-09T06:45:55.104696+0000 mon.a (mon.0) 1557 : cluster 0 osdmap e298: 8 total, 8 up, 4 in 2023-10-09T06:45:55.842 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:55.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:55.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:55.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:56.082 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.109 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:56.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:45:56.135144+0000 min lat: 0.00130822 max lat: 0.364762 avg lat: 0.00942017 2023-10-09T06:45:56.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:45:56.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 20 16 33967 33951 106.087 77.0625 0.0211073 0.00942017 2023-10-09T06:45:56.212 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:56.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:56.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:56.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:56.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:56.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:56.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:56 smithi125 bash[80002]: cluster 2023-10-09T06:45:55.103684+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T06:45:56.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:56 smithi125 bash[80002]: cluster 2023-10-09T06:45:55.401793+0000 mgr.y (mgr.25723) 1240 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:56.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:56.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:56.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:56.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:56 smithi125 bash[77299]: cluster 2023-10-09T06:45:55.103684+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T06:45:56.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:56 smithi125 bash[77299]: cluster 2023-10-09T06:45:55.401793+0000 mgr.y (mgr.25723) 1240 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:56.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:56.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:56 smithi155 bash[59495]: cluster 2023-10-09T06:45:55.103684+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e298 with expected crc 2023-10-09T06:45:56.882 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:56 smithi155 bash[59495]: cluster 2023-10-09T06:45:55.401793+0000 mgr.y (mgr.25723) 1240 : cluster 0 pgmap v1345: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:56.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:56.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:57.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:57.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 21 16 35154 35138 104.568 74.1875 0.0183216 0.00955627 2023-10-09T06:45:57.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.170 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:57.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:57.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:57.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:57.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:57.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:57.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:57.662 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:57.686 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:57.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:57.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:57.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:57.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:58.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:58.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:58.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 22 16 36478 36462 103.576 82.75 0.014028 0.00964894 2023-10-09T06:45:58.178 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:58.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:45:58.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:45:58.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:58.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:58.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:58.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:58.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:58.477 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:45:58 smithi125 bash[77299]: cluster 2023-10-09T06:45:57.402155+0000 mgr.y (mgr.25723) 1241 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:58.477 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:45:58 smithi125 bash[80002]: cluster 2023-10-09T06:45:57.402155+0000 mgr.y (mgr.25723) 1241 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:58.565 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:58.588 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:58.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:45:58 smithi155 bash[59495]: cluster 2023-10-09T06:45:57.402155+0000 mgr.y (mgr.25723) 1241 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.0 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:45:58.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:58.714 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:58.819 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:58.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:58.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:58.945 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:58.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:45:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:58.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:45:59.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:59.106 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:45:59.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:45:59.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 23 16 37715 37699 102.434 77.3125 0.00947217 0.0097559 2023-10-09T06:45:59.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:59.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:45:59.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:45:59.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:59.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:45:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:45:59.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:59.489 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:59.592 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:59.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:45:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:45:59.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:59.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:45:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:45:59.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:59.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:45:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:45:59.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:45:59.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:00.015 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:00.125 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:00.135 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:00 smithi155 bash[59495]: cluster 2023-10-09T06:45:59.402717+0000 mgr.y (mgr.25723) 1242 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:46:00.138 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 24 16 38988 38972 101.48 79.5625 0.025489 0.00984598 2023-10-09T06:46:00.155 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:00.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:00.273 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:00 smithi125 bash[77299]: cluster 2023-10-09T06:45:59.402717+0000 mgr.y (mgr.25723) 1242 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:46:00.273 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:00 smithi125 bash[80002]: cluster 2023-10-09T06:45:59.402717+0000 mgr.y (mgr.25723) 1242 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:46:00.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:00.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:00.422 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:00.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:00.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:00.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:00.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:00.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:00.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:00.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:00.963 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:00.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:01.065 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:01.085 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:01.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 25 16 40171 40155 100.378 73.9375 0.00881769 0.00995553 2023-10-09T06:46:01.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:01.310 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.337 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:01] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-10-09T06:46:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:01.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:01.448 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:46:01.464 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:01.567 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:01.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:01.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:01.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:01.827 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:01.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:01.960 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:01.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:01.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:02.088 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:02.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:02.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:02.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 26 16 41414 41398 99.5054 77.6875 0.0101867 0.0100432 2023-10-09T06:46:02.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:02.264 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:02.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:02.401 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:02.506 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:02.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:02.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:02.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:02 smithi155 bash[59495]: cluster 2023-10-09T06:46:01.403609+0000 mgr.y (mgr.25723) 1243 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:02.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:02.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:02.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:02.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:02 smithi125 bash[77299]: cluster 2023-10-09T06:46:01.403609+0000 mgr.y (mgr.25723) 1243 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:02.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:02 smithi125 bash[80002]: cluster 2023-10-09T06:46:01.403609+0000 mgr.y (mgr.25723) 1243 : cluster 0 pgmap v1348: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:02.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:02.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:02.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:03.033 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:03.056 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:03.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 27 16 42620 42604 98.6114 75.375 0.0100531 0.0101339 2023-10-09T06:46:03.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.183 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:03.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:03.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:03.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:03.488 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:03.542 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:03.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:03.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:03.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:03.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:03.821 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:03.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:03.952 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:03.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:04.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:04.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:04.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:04.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 28 16 43811 43795 97.7478 74.4375 0.0248221 0.0102232 2023-10-09T06:46:04.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:04.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:04.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:04.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:04.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:04.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:04.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[77299]: cluster 2023-10-09T06:46:03.404128+0000 mgr.y (mgr.25723) 1244 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:04.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[77299]: audit 2023-10-09T06:46:03.610638+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:04.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[77299]: audit 2023-10-09T06:46:03.612098+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:04.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:04.593 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:04.601 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:04 smithi155 bash[59495]: cluster 2023-10-09T06:46:03.404128+0000 mgr.y (mgr.25723) 1244 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:04.601 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:04 smithi155 bash[59495]: audit 2023-10-09T06:46:03.610638+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:04.602 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:04 smithi155 bash[59495]: audit 2023-10-09T06:46:03.612098+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:04.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:04.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:04.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[80002]: cluster 2023-10-09T06:46:03.404128+0000 mgr.y (mgr.25723) 1244 : cluster 0 pgmap v1349: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:46:04.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[80002]: audit 2023-10-09T06:46:03.610638+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:04.731 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:04 smithi125 bash[80002]: audit 2023-10-09T06:46:03.612098+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:04.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:04.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:04.876 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:04.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:04.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:05.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:05.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:05.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 29 16 44907 44891 96.7391 68.5 0.0166105 0.010329 2023-10-09T06:46:05.139 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:05.248 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:05.272 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:05.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:05.374 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:46:05.374 INFO:teuthology.orchestra.run.smithi125.stdout: "success": "" 2023-10-09T06:46:05.374 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:46:05.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:05.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:05.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:05.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:05.625 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:05.663 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:05.684 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:05.787 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:05.815 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:05.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:05.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:05.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:06.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:06.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:06.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 30 16 46006 45990 95.8038 68.6875 0.0209101 0.0104321 2023-10-09T06:46:06.192 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:06.318 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:06.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:06.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:06.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:06.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:06.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:06.748 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:06.753 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:06 smithi155 bash[59495]: cluster 2023-10-09T06:46:05.404772+0000 mgr.y (mgr.25723) 1245 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:46:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:06.855 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:06 smithi125 bash[77299]: cluster 2023-10-09T06:46:05.404772+0000 mgr.y (mgr.25723) 1245 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:46:06.865 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:06 smithi125 bash[80002]: cluster 2023-10-09T06:46:05.404772+0000 mgr.y (mgr.25723) 1245 : cluster 0 pgmap v1350: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:46:06.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:06.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:06.986 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:07.013 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:07.118 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:07.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 31 16 47112 47096 94.9429 69.125 0.0230669 0.0105226 2023-10-09T06:46:07.140 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:07.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:07.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:07.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:07.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:07.508 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:07.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:07.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:07.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:07.763 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:07.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:07.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:07.920 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:07.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:08.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:08.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 32 16 48185 48169 94.0714 67.0625 0.0115685 0.0106239 2023-10-09T06:46:08.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:08.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:08.290 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:08.320 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:08.423 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:08.447 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:08.552 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.573 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:08.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:08 smithi125 bash[80002]: cluster 2023-10-09T06:46:07.405265+0000 mgr.y (mgr.25723) 1246 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:08.627 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:46:08.628 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:46:08.676 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:08.703 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:08.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:08.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:08.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:08 smithi125 bash[77299]: cluster 2023-10-09T06:46:07.405265+0000 mgr.y (mgr.25723) 1246 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:08.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:08 smithi155 bash[59495]: cluster 2023-10-09T06:46:07.405265+0000 mgr.y (mgr.25723) 1246 : cluster 0 pgmap v1351: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:08.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:08.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:08.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:09.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:09.088 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:09.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 33 16 49251 49235 93.2395 66.625 0.0175343 0.0107198 2023-10-09T06:46:09.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:09.215 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:09.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:09.343 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:09.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:09.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:09.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:09.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:09.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:09.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:09.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:09.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:09.841 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:09.864 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:09.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:09.996 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:09.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:09.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:10.099 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:10.137 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:10.139 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 34 16 50130 50114 92.1128 54.9375 0.0237313 0.0108506 2023-10-09T06:46:10.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:10.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:10.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.378 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:10 smithi125 bash[77299]: cluster 2023-10-09T06:46:09.405644+0000 mgr.y (mgr.25723) 1247 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:10.379 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:10 smithi125 bash[80002]: cluster 2023-10-09T06:46:09.405644+0000 mgr.y (mgr.25723) 1247 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:10 smithi155 bash[59495]: cluster 2023-10-09T06:46:09.405644+0000 mgr.y (mgr.25723) 1247 : cluster 0 pgmap v1352: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:46:10.394 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:10.497 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:10.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:10.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:10.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:10.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:10.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:10.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:10.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:11.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:11.036 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:11.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 35 16 51255 51239 91.4897 70.3125 0.00765871 0.010925 2023-10-09T06:46:11.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:11.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:11.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:11.297 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:11.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.432 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:11.483 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:11] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-10-09T06:46:11.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:11.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:11.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:11.690 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:11.793 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:11.936 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.965 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:11.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:12.069 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:12.096 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:12.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 36 16 52309 52293 90.778 65.875 0.0125085 0.0110106 2023-10-09T06:46:12.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:12.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:12.332 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:12.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:12.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:12.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:12.592 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:12.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:12.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:12 smithi155 bash[59495]: cluster 2023-10-09T06:46:11.406284+0000 mgr.y (mgr.25723) 1248 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:46:12.667 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:12 smithi125 bash[77299]: cluster 2023-10-09T06:46:11.406284+0000 mgr.y (mgr.25723) 1248 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:46:12.668 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:12 smithi125 bash[80002]: cluster 2023-10-09T06:46:11.406284+0000 mgr.y (mgr.25723) 1248 : cluster 0 pgmap v1353: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-10-09T06:46:12.716 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:12.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:12.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:12.878 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:12.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:12.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:13.007 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:13.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:13.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:13.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:13.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 37 16 53278 53262 89.9613 60.5625 0.0269678 0.0110995 2023-10-09T06:46:13.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:13.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:13.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:13.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.536 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:13.641 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:13.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:13.782 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:13.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:13.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:13.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:13.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:14.045 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:14.067 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:14.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 38 16 54265 54249 89.2171 61.6875 0.0100333 0.0112024 2023-10-09T06:46:14.171 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:14.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:14.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:14.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:14.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:14.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:14.468 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:14.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:14.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:46:14.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:14.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:14.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:14.755 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:14 smithi125 bash[77299]: cluster 2023-10-09T06:46:13.406673+0000 mgr.y (mgr.25723) 1249 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:46:14.756 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:14 smithi125 bash[80002]: cluster 2023-10-09T06:46:13.406673+0000 mgr.y (mgr.25723) 1249 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:46:14.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:14.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:14.884 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:14 smithi155 bash[59495]: cluster 2023-10-09T06:46:13.406673+0000 mgr.y (mgr.25723) 1249 : cluster 0 pgmap v1354: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:46:14.904 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:14.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:15.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:15.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:15.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:15.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 39 16 55294 55278 88.5784 64.3125 0.0125574 0.0112838 2023-10-09T06:46:15.165 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:15.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:15.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:15.411 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:15.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:15.538 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:15.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:15.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:15.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:15.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:15.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:15.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:15.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:15.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:15.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:16.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:16.089 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: audit 2023-10-09T06:46:14.761854+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: audit 2023-10-09T06:46:14.768571+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: audit 2023-10-09T06:46:14.769835+0000 mon.a (mon.0) 1562 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: audit 2023-10-09T06:46:14.771064+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: audit 2023-10-09T06:46:14.777407+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[77299]: cluster 2023-10-09T06:46:15.407482+0000 mgr.y (mgr.25723) 1250 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: audit 2023-10-09T06:46:14.761854+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: audit 2023-10-09T06:46:14.768571+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: audit 2023-10-09T06:46:14.769835+0000 mon.a (mon.0) 1562 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:16.090 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: audit 2023-10-09T06:46:14.771064+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:16.091 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: audit 2023-10-09T06:46:14.777407+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.091 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:15 smithi125 bash[80002]: cluster 2023-10-09T06:46:15.407482+0000 mgr.y (mgr.25723) 1250 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:46:16.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:16.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:46:16.137019+0000 min lat: 0.00130822 max lat: 0.364762 avg lat: 0.0113645 2023-10-09T06:46:16.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:46:16.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 40 16 56304 56288 87.9419 63.125 0.0244371 0.0113645 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: audit 2023-10-09T06:46:14.761854+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: audit 2023-10-09T06:46:14.768571+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: audit 2023-10-09T06:46:14.769835+0000 mon.a (mon.0) 1562 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: audit 2023-10-09T06:46:14.771064+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: audit 2023-10-09T06:46:14.777407+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:16.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:15 smithi155 bash[59495]: cluster 2023-10-09T06:46:15.407482+0000 mgr.y (mgr.25723) 1250 : cluster 0 pgmap v1355: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-10-09T06:46:16.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:16.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:16.373 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:16.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.484 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 7, 6] out_osds: [5, 1, 2, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:46:16.485 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:46:16.485 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:46:16.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:46:16.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:16.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:16.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:16.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:16.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:16.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:16.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:17.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:17.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:17.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:17.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 41 16 57293 57277 87.3044 61.8125 0.0138829 0.0114471 2023-10-09T06:46:17.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:17.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:17.296 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:17.402 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:17.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:17.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:17.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:17.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:17.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:17.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:17.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:17.930 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:17.966 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:17.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:18.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:18.099 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:18.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 42 16 58310 58294 86.739 63.5625 0.0149074 0.0115236 2023-10-09T06:46:18.202 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:18.229 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:18.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:18.465 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:18.467 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:18 smithi155 bash[59495]: cluster 2023-10-09T06:46:17.408048+0000 mgr.y (mgr.25723) 1251 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T06:46:18.513 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:18.617 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.625 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:18 smithi125 bash[77299]: cluster 2023-10-09T06:46:17.408048+0000 mgr.y (mgr.25723) 1251 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T06:46:18.626 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:18 smithi125 bash[80002]: cluster 2023-10-09T06:46:17.408048+0000 mgr.y (mgr.25723) 1251 : cluster 0 pgmap v1356: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-09T06:46:18.641 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:18.745 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:18.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:18.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:18.999 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:19.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:19.140 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:19.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 43 16 59318 59302 86.1868 63 0.0129589 0.0115965 2023-10-09T06:46:19.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:19.261 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:46:19.282 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:19.288 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:46:19.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:19.317 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:46:19.342 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:46:19.373 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:46:19.398 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:46:19.409 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:19.436 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:46:19.448 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:19.466 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:46:19.469 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.469 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.469 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.469 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.470 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.470 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.470 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.470 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:46:19.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:19.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:19.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:19 smithi125 bash[77299]: audit 2023-10-09T06:46:18.607422+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:19.611 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:19 smithi125 bash[77299]: audit 2023-10-09T06:46:19.228202+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.125:0/527244063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:19.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:19 smithi125 bash[80002]: audit 2023-10-09T06:46:18.607422+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:19.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:19 smithi125 bash[80002]: audit 2023-10-09T06:46:19.228202+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.125:0/527244063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:19.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:19.710 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:19 smithi155 bash[59495]: audit 2023-10-09T06:46:18.607422+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:19.710 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:19 smithi155 bash[59495]: audit 2023-10-09T06:46:19.228202+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.125:0/527244063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:19.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:19.836 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:19.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:19.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:19.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:20.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:20.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:20.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:20.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 44 15 60300 60285 85.6242 61.4375 0.0187902 0.0116728 2023-10-09T06:46:20.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:20.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:20.372 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.396 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:20.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:20.522 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:20.625 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:20 smithi125 bash[77299]: cluster 2023-10-09T06:46:19.408612+0000 mgr.y (mgr.25723) 1252 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:46:20.633 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:20 smithi125 bash[80002]: cluster 2023-10-09T06:46:19.408612+0000 mgr.y (mgr.25723) 1252 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:46:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:20 smithi155 bash[59495]: cluster 2023-10-09T06:46:19.408612+0000 mgr.y (mgr.25723) 1252 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 869 op/s 2023-10-09T06:46:20.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:20.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.776 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:20.879 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:20.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:20.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:21.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:21.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:21.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:21.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 45 16 61292 61276 85.0977 61.9375 0.0130502 0.011746 2023-10-09T06:46:21.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:21.258 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:21.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:21.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:21.402 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:21] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.33.4" 2023-10-09T06:46:21.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:21.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:21.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:21.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:21.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:21.754 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:21.755 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:21.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:21.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:21.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:21.964 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:21.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:21.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:22.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:22.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:22.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 46 16 62254 62238 84.5547 60.125 0.0140972 0.0118211 2023-10-09T06:46:22.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:22.216 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:22.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:22.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:22.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:22.466 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:22.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:22.479 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:46:22.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:22 smithi155 bash[59495]: cluster 2023-10-09T06:46:21.409419+0000 mgr.y (mgr.25723) 1253 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:46:22.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:22 smithi155 bash[59495]: audit 2023-10-09T06:46:21.705639+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.125:0/3356737383' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:22 smithi155 bash[59495]: audit 2023-10-09T06:46:21.706036+0000 mon.a (mon.0) 1566 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.569 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:22.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[77299]: cluster 2023-10-09T06:46:21.409419+0000 mgr.y (mgr.25723) 1253 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:46:22.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[77299]: audit 2023-10-09T06:46:21.705639+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.125:0/3356737383' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.577 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[77299]: audit 2023-10-09T06:46:21.706036+0000 mon.a (mon.0) 1566 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[80002]: cluster 2023-10-09T06:46:21.409419+0000 mgr.y (mgr.25723) 1253 : cluster 0 pgmap v1358: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T06:46:22.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[80002]: audit 2023-10-09T06:46:21.705639+0000 mon.c (mon.2) 120 : audit 1 from='client.? 172.21.15.125:0/3356737383' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.578 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:22 smithi125 bash[80002]: audit 2023-10-09T06:46:21.706036+0000 mon.a (mon.0) 1566 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:46:22.591 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:22.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:22.695 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:22.718 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:22.821 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:22.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:22.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:22.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:22.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:22.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:23.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:23.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:23.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 47 16 63197 63181 84.0096 58.9375 0.019883 0.0118974 2023-10-09T06:46:23.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:23.233 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:23.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:46:23.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:23.361 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:23.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:23.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:23.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[77299]: cluster 2023-10-09T06:46:22.466173+0000 mon.a (mon.0) 1567 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[77299]: cluster 2023-10-09T06:46:22.470054+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[77299]: cluster 2023-10-09T06:46:22.470298+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[77299]: audit 2023-10-09T06:46:22.470474+0000 mon.a (mon.0) 1568 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[77299]: cluster 2023-10-09T06:46:22.470589+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e299: 8 total, 8 up, 4 in 2023-10-09T06:46:23.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[80002]: cluster 2023-10-09T06:46:22.466173+0000 mon.a (mon.0) 1567 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:23.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[80002]: cluster 2023-10-09T06:46:22.470054+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[80002]: cluster 2023-10-09T06:46:22.470298+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[80002]: audit 2023-10-09T06:46:22.470474+0000 mon.a (mon.0) 1568 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:46:23.611 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:23 smithi125 bash[80002]: cluster 2023-10-09T06:46:22.470589+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e299: 8 total, 8 up, 4 in 2023-10-09T06:46:23.624 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:23.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:23.736 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:23 smithi155 bash[59495]: cluster 2023-10-09T06:46:22.466173+0000 mon.a (mon.0) 1567 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:23.736 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:23 smithi155 bash[59495]: cluster 2023-10-09T06:46:22.470054+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.736 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:23 smithi155 bash[59495]: cluster 2023-10-09T06:46:22.470298+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e299 with expected crc 2023-10-09T06:46:23.736 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:23 smithi155 bash[59495]: audit 2023-10-09T06:46:22.470474+0000 mon.a (mon.0) 1568 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:46:23.736 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:23 smithi155 bash[59495]: cluster 2023-10-09T06:46:22.470589+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e299: 8 total, 8 up, 4 in 2023-10-09T06:46:23.757 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:23.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:23.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:23.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:23.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:24.020 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:24.123 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:24.140 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 48 16 64132 64116 83.4767 58.4375 0.0106728 0.0119738 2023-10-09T06:46:24.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:24.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:24.254 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:24.287 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:24.393 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:24.419 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:24.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:24.532 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:24 smithi155 bash[59495]: cluster 2023-10-09T06:46:23.409824+0000 mgr.y (mgr.25723) 1254 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:46:24.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:24.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:24.679 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:24 smithi125 bash[77299]: cluster 2023-10-09T06:46:23.409824+0000 mgr.y (mgr.25723) 1254 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:46:24.679 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:24 smithi125 bash[80002]: cluster 2023-10-09T06:46:23.409824+0000 mgr.y (mgr.25723) 1254 : cluster 0 pgmap v1360: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T06:46:24.695 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:24.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:24.830 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:24.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:24.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:24.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:25.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.101 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:25.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 49 16 65071 65055 82.9708 58.6875 0.0209575 0.0120468 2023-10-09T06:46:25.206 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:25.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:25.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:25.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:25.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:25.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.504 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:25.607 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:25.648 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:25.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:25.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:25.884 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.914 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:25.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:26.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:26.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 50 16 65998 65982 82.47 57.9375 0.0142123 0.0121196 2023-10-09T06:46:26.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:26.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:26.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:26.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:26.401 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.438 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:26.491 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:26 smithi125 bash[77299]: cluster 2023-10-09T06:46:25.410760+0000 mgr.y (mgr.25723) 1255 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:26.543 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.585 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:26.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:26.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:26 smithi125 bash[80002]: cluster 2023-10-09T06:46:25.410760+0000 mgr.y (mgr.25723) 1255 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:26.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:26.697 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:26 smithi155 bash[59495]: cluster 2023-10-09T06:46:25.410760+0000 mgr.y (mgr.25723) 1255 : cluster 0 pgmap v1361: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:26.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:26.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:26.861 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:26.964 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:26.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:26.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:27.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:46:27.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:27.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:27.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 51 16 66711 66695 81.7267 44.5625 0.0195429 0.0122301 2023-10-09T06:46:27.218 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.248 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:27.353 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:27.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:27.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:27.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:27.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:27.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:27.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:27.756 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:27.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:27.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:27.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:28.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:28.062 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:28.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 52 16 67607 67591 81.2318 56 0.0285453 0.012304 2023-10-09T06:46:28.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:28.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:28.301 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:28.329 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:28.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:28.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:28.458 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:28.561 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:28.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:28 smithi125 bash[77299]: cluster 2023-10-09T06:46:27.411221+0000 mgr.y (mgr.25723) 1256 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:28.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:28 smithi125 bash[80002]: cluster 2023-10-09T06:46:27.411221+0000 mgr.y (mgr.25723) 1256 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:28.589 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:28.692 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:28.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:28.825 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:28.847 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:28.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:28 smithi155 bash[59495]: cluster 2023-10-09T06:46:27.411221+0000 mgr.y (mgr.25723) 1256 : cluster 0 pgmap v1362: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:28.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:28.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:28.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:29.076 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:29.101 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:29.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:29.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 53 16 68514 68498 80.7686 56.6875 0.0155841 0.0123751 2023-10-09T06:46:29.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:29.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:29.333 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:29.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:29.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:29.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:29.585 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:46:29.585 INFO:teuthology.orchestra.run.smithi125.stdout: "size": 20, 2023-10-09T06:46:29.585 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 600, 2023-10-09T06:46:29.585 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [ 2023-10-09T06:46:29.585 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.590 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209876 8.10 8:0ac997c0:::benchmark_data_smithi125_176369_object100808:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20398188199999, 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.526103038, 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "reached pg", 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.591 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209876 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.592 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372667+0000", 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372665+0000", 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999986 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.593 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372672+0000", 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.5570000000000001e-06 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372684+0000", 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.2084e-05 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372694+0000", 2023-10-09T06:46:29.594 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0066e-05 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372742+0000", 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8423e-05 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372752+0000", 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7310000000000005e-06 2023-10-09T06:46:29.595 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.898696+0000", 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5259440899999999 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.596 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.898770+0000", 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3543999999999999e-05 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209923 8.10 8:0959915d:::benchmark_data_smithi125_176369_object100855:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20394321200001, 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5373261359999999, 2023-10-09T06:46:29.597 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209923 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.598 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372705+0000", 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.599 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372708+0000", 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372715+0000", 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.9500000000000004e-06 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.600 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372719+0000", 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4910000000000001e-06 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372790+0000", 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1585e-05 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.601 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0857e-05 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899097+0000", 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.526296098 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899208+0000", 2023-10-09T06:46:29.602 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000111191 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.899512+0000", 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030379499999999999 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906408+0000", 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0068957419999999998 2023-10-09T06:46:29.603 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906428+0000", 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.0163e-05 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906468+0000", 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9363000000000002e-05 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.604 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906654+0000", 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000186866 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909813+0000", 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0031590680000000001 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.605 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909888+0000", 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4727000000000005e-05 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910031+0000", 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014324199999999999 2023-10-09T06:46:29.606 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209926 8.10 8:0d73ff13:::benchmark_data_smithi125_176369_object100858:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20391328599999, 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5375148360000002, 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.607 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209926 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.608 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372735+0000", 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.609 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372738+0000", 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.993e-06 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372747+0000", 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.3850000000000008e-06 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.610 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372753+0000", 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.879e-06 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372835+0000", 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.1921000000000004e-05 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.611 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372842+0000", 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5039999999999996e-06 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901234+0000", 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.528391244 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.612 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901444+0000", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020991599999999999 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.901862+0000", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041795800000000001 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906442+0000", 2023-10-09T06:46:29.613 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045809750000000002 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906699+0000", 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000256796 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906785+0000", 2023-10-09T06:46:29.614 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6135000000000001e-05 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909564+0000", 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0027786479999999999 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910082+0000", 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00051772999999999999 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.615 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910130+0000", 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8482000000000001e-05 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910250+0000", 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000119752 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.616 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209934 8.10 8:0cf8a368:::benchmark_data_smithi125_176369_object100866:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20389190499998, 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378215150000001, 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.617 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209934 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.618 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372757+0000", 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372756+0000", 2023-10-09T06:46:29.619 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372759+0000", 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.8830000000000002e-06 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372764+0000", 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.0900000000000004e-06 2023-10-09T06:46:29.620 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372772+0000", 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3839999999999999e-06 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372878+0000", 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000106623 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.621 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372889+0000", 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0725e-05 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904119+0000", 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5312299970000001 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.622 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904293+0000", 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000173952 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.904581+0000", 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000288206 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.623 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906670+0000", 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0020891759999999999 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906845+0000", 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017475499999999999 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.624 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909478+0000", 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0026327379999999999 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.909651+0000", 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017286600000000001 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.625 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910359+0000", 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00070834399999999997 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910409+0000", 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9749000000000001e-05 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.626 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910578+0000", 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016941100000000001 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209937 8.10 8:0a4d40e2:::benchmark_data_smithi125_176369_object100869:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:46:29.627 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20386621800003, 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.551965877, 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209937 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.628 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.629 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372782+0000", 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372784+0000", 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6089999999999999e-06 2023-10-09T06:46:29.630 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372790+0000", 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9510000000000003e-06 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372796+0000", 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.259e-06 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.631 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372923+0000", 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012774100000000001 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372928+0000", 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6020000000000002e-06 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.632 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.906896+0000", 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5339678559999999 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.907123+0000", 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022763699999999999 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.907470+0000", 2023-10-09T06:46:29.633 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034701399999999998 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910323+0000", 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0028525870000000002 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.634 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910619+0000", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029599100000000001 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.916236+0000", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0056167999999999999 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920508+0000", 2023-10-09T06:46:29.635 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.004272395 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924535+0000", 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0040265850000000001 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924577+0000", 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2577000000000002e-05 2023-10-09T06:46:29.636 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924748+0000", 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017075599999999999 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209941 8.10 8:09a16055:::benchmark_data_smithi125_176369_object100873:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.637 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20384777300001, 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5561017150000001, 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.638 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209941 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372801+0000", 2023-10-09T06:46:29.639 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372800+0000", 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372803+0000", 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.7140000000000002e-06 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.640 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372809+0000", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.8329999999999999e-06 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372815+0000", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4649999999999999e-06 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372967+0000", 2023-10-09T06:46:29.641 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015147199999999999 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372974+0000", 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8720000000000002e-06 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910799+0000", 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5378241350000001 2023-10-09T06:46:29.642 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.910924+0000", 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012560300000000001 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.911303+0000", 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000379162 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.643 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.916296+0000", 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0049930189999999996 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920533+0000", 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0042367350000000002 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.644 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920555+0000", 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1563999999999999e-05 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920609+0000", 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4772999999999999e-05 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.645 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928730+0000", 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0081201290000000002 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928768+0000", 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8661000000000001e-05 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.646 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928902+0000", 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013415599999999999 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209952 8.10 8:0d737c89:::benchmark_data_smithi125_176369_object100884:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20381524799996, 2023-10-09T06:46:29.647 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5562271050000001, 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209952 2023-10-09T06:46:29.648 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.649 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372833+0000", 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372835+0000", 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3640000000000002e-06 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.650 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372840+0000", 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372845+0000", 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2810000000000002e-06 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.651 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373007+0000", 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00016230199999999999 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.652 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373012+0000", 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.527e-06 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.653 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.913584+0000", 2023-10-09T06:46:29.654 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.540572426 2023-10-09T06:46:29.654 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.654 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.654 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.654 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.913740+0000", 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015646900000000001 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.914050+0000", 2023-10-09T06:46:29.655 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00030994799999999999 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920577+0000", 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0065267800000000003 2023-10-09T06:46:29.656 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920632+0000", 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.4620999999999998e-05 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.657 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.658 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.658 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920654+0000", 2023-10-09T06:46:29.658 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.1880999999999999e-05 2023-10-09T06:46:29.658 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.658 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924525+0000", 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0038716390000000001 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.659 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.660 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928916+0000", 2023-10-09T06:46:29.660 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0043907859999999998 2023-10-09T06:46:29.660 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.660 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.660 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.661 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928950+0000", 2023-10-09T06:46:29.661 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.3986999999999998e-05 2023-10-09T06:46:29.661 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.661 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.661 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929060+0000", 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000110287 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.662 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.663 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.663 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209980 8.10 8:08bfd313:::benchmark_data_smithi125_176369_object100912:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.663 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:46:29.663 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20380072099999, 2023-10-09T06:46:29.663 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5691573399999998, 2023-10-09T06:46:29.664 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.664 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.664 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209980 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.665 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:46:29.666 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.666 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.666 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.666 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.666 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372848+0000", 2023-10-09T06:46:29.667 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.667 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.667 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.667 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.667 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372847+0000", 2023-10-09T06:46:29.668 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.668 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.668 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.668 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.668 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372850+0000", 2023-10-09T06:46:29.669 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5100000000000001e-06 2023-10-09T06:46:29.669 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.669 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.669 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.669 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372856+0000", 2023-10-09T06:46:29.670 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.212e-06 2023-10-09T06:46:29.670 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.670 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.670 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.670 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372863+0000", 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7660000000000001e-06 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373048+0000", 2023-10-09T06:46:29.671 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000185415 2023-10-09T06:46:29.672 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.672 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.672 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.672 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373056+0000", 2023-10-09T06:46:29.672 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5480000000000004e-06 2023-10-09T06:46:29.673 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.673 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.673 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.673 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920669+0000", 2023-10-09T06:46:29.673 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.54761273 2023-10-09T06:46:29.674 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.674 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.674 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.674 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.920766+0000", 2023-10-09T06:46:29.674 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7446000000000002e-05 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.921042+0000", 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027558299999999998 2023-10-09T06:46:29.675 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.928717+0000", 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0076757220000000003 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.676 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929109+0000", 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00039157699999999999 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929159+0000", 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.9985999999999999e-05 2023-10-09T06:46:29.677 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929235+0000", 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6007999999999995e-05 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.678 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.941816+0000", 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012580780999999999 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.941880+0000", 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3804000000000001e-05 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.679 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942005+0000", 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012555800000000001 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.680 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:209985 8.10 8:09a3bec9:::benchmark_data_smithi125_176369_object100917:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20378064600004, 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.579772967, 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.681 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 209985 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.682 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372868+0000", 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.683 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372867+0000", 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372870+0000", 2023-10-09T06:46:29.684 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5059999999999998e-06 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372882+0000", 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.1892000000000001e-05 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.685 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372892+0000", 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.0473e-05 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373084+0000", 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000192086 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.686 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373089+0000", 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.566e-06 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.687 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922465+0000", 2023-10-09T06:46:29.688 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.549376004 2023-10-09T06:46:29.688 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.688 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.688 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.688 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922643+0000", 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017813599999999999 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.922860+0000", 2023-10-09T06:46:29.689 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00021646400000000001 2023-10-09T06:46:29.690 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.690 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.690 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.690 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929145+0000", 2023-10-09T06:46:29.690 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0062852359999999996 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929168+0000", 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3047000000000001e-05 2023-10-09T06:46:29.691 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.692 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.692 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.692 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942069+0000", 2023-10-09T06:46:29.692 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.01290088 2023-10-09T06:46:29.692 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952399+0000", 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010330548 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.693 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.694 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.694 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952440+0000", 2023-10-09T06:46:29.694 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.07e-05 2023-10-09T06:46:29.694 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.694 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.695 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.695 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952493+0000", 2023-10-09T06:46:29.695 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3449999999999998e-05 2023-10-09T06:46:29.695 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.695 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.696 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.696 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952641+0000", 2023-10-09T06:46:29.696 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014738500000000001 2023-10-09T06:46:29.696 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.696 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210039 8.10 8:0c2bb4d9:::benchmark_data_smithi125_176369_object100971:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:46:29.697 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20373937399995, 2023-10-09T06:46:29.698 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5800452730000001, 2023-10-09T06:46:29.698 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.698 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.698 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.699 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.699 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.699 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210039 2023-10-09T06:46:29.699 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.699 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.700 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.700 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.700 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:46:29.700 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.700 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372909+0000", 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.701 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372911+0000", 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.187e-06 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372915+0000", 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2830000000000003e-06 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.702 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372919+0000", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5810000000000001e-06 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373122+0000", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000203628 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373130+0000", 2023-10-09T06:46:29.703 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.3849999999999996e-06 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924795+0000", 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.551665587 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.924966+0000", 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017071999999999999 2023-10-09T06:46:29.704 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.925225+0000", 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025892899999999997 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929200+0000", 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.003974712 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.705 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929245+0000", 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.4753000000000003e-05 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942110+0000", 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012865639 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.706 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952687+0000", 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010576962000000001 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952779+0000", 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1593000000000002e-05 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952826+0000", 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6751e-05 2023-10-09T06:46:29.707 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952954+0000", 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012888600000000001 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210147 8.10 8:0a1f5bb6:::benchmark_data_smithi125_176369_object101079:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:46:29.708 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20372689700002, 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5803002130000001, 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210147 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.709 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372922+0000", 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372921+0000", 2023-10-09T06:46:29.710 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967296 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372923+0000", 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.7179999999999999e-06 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372926+0000", 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.128e-06 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.711 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372929+0000", 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.858e-06 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373158+0000", 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022874200000000001 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.712 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373163+0000", 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.8729999999999998e-06 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.926856+0000", 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.553693335 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.927033+0000", 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00017658100000000001 2023-10-09T06:46:29.713 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.927286+0000", 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025329999999999998 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929252+0000", 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0019665669999999998 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.714 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933820+0000", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0045678740000000004 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942144+0000", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0083233260000000007 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953010+0000", 2023-10-09T06:46:29.715 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010866752 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953051+0000", 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.0169000000000001e-05 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953093+0000", 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2818000000000002e-05 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.716 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953222+0000", 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000128405 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210173 8.10 8:0e3f8787:::benchmark_data_smithi125_176369_object101105:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20371536899995, 2023-10-09T06:46:29.717 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5804989599999999, 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210173 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.718 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372933+0000", 2023-10-09T06:46:29.719 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372939+0000", 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.2609999999999999e-06 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372949+0000", 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.7850000000000003e-06 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.720 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372955+0000", 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.4250000000000003e-06 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373195+0000", 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023943200000000001 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.721 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373202+0000", 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1439999999999997e-06 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929278+0000", 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5560760010000001 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929364+0000", 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.6339000000000001e-05 2023-10-09T06:46:29.722 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.929557+0000", 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00019301499999999999 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942023+0000", 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012465385000000001 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.723 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942179+0000", 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015596100000000001 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952230+0000", 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010051480999999999 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.724 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953256+0000", 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0010258730000000001 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953285+0000", 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.9149000000000002e-05 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953324+0000", 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9001999999999997e-05 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.725 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953432+0000", 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010805400000000001 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210227 8.10 8:0feaf1d8:::benchmark_data_smithi125_176369_object101159:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:46:29.726 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20368775500003, 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5807789350000001, 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210227 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.727 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372961+0000", 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.728 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372960+0000", 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372963+0000", 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.6740000000000001e-06 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372970+0000", 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.764e-06 2023-10-09T06:46:29.729 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372975+0000", 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.1429999999999997e-06 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373228+0000", 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025312799999999998 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.730 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373232+0000", 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.3100000000000002e-06 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931483+0000", 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.558250446 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.731 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931571+0000", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.8069e-05 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.931810+0000", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00023871799999999999 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942029+0000", 2023-10-09T06:46:29.732 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010219587 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942215+0000", 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00018612699999999999 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952275+0000", 2023-10-09T06:46:29.733 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010060176000000001 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953466+0000", 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001190705 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953591+0000", 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000124697 2023-10-09T06:46:29.734 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953630+0000", 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.9629999999999998e-05 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953740+0000", 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000109197 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.735 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210237 8.10 8:0d109371:::benchmark_data_smithi125_176369_object101169:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20366890599996, 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.581022412, 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.736 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210237 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372980+0000", 2023-10-09T06:46:29.737 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372979+0000", 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372982+0000", 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.3690000000000001e-06 2023-10-09T06:46:29.738 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372988+0000", 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.7800000000000003e-06 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.372996+0000", 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1999999999999997e-06 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.739 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373253+0000", 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025763499999999998 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373257+0000", 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.7749999999999999e-06 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.740 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933851+0000", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5605936740000002 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.933991+0000", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013992599999999999 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.934269+0000", 2023-10-09T06:46:29.741 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027870999999999999 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942076+0000", 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0078070090000000002 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952316+0000", 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010239738999999999 2023-10-09T06:46:29.742 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953497+0000", 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0011805979999999999 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953527+0000", 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0144000000000001e-05 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.743 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953773+0000", 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024658699999999999 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953886+0000", 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000112611 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.744 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954002+0000", 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000116005 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210258 8.10 8:099246c7:::benchmark_data_smithi125_176369_object101190:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20364418899999, 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.581382085, 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.745 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210258 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:46:29.746 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373004+0000", 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.747 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373006+0000", 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.2180000000000002e-06 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373010+0000", 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8800000000000001e-06 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.748 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373014+0000", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.391e-06 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373278+0000", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000264054 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373283+0000", 2023-10-09T06:46:29.749 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.5530000000000003e-06 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939405+0000", 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5661224279999999 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939560+0000", 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00015436000000000001 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.750 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.939857+0000", 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029764599999999999 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952357+0000", 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.012499962999999999 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.751 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952967+0000", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00060953199999999998 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.953561+0000", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00059446399999999995 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954039+0000", 2023-10-09T06:46:29.752 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00047749899999999999 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954188+0000", 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014903200000000001 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954274+0000", 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 8.5918e-05 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.753 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954386+0000", 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00011252 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:210309 8.10 8:0c45fd6a:::benchmark_data_smithi125_176369_object101241:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:46:29.754 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 517.20362249899995, 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5916112199999999, 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 210309 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.755 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373026+0000", 2023-10-09T06:46:29.756 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373028+0000", 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.65e-06 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373036+0000", 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.5680000000000002e-06 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.757 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373042+0000", 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.7710000000000003e-06 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373296+0000", 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00025467900000000001 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.758 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:52.373300+0000", 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.636e-06 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942243+0000", 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.5689430880000002 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942373+0000", 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00013034900000000001 2023-10-09T06:46:29.759 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 3,4,5,7", 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.942714+0000", 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00034058500000000002 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.952974+0000", 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010259727 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.760 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954070+0000", 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001095993 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954129+0000", 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.9283000000000001e-05 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.761 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.954422+0000", 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000292916 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964457+0000", 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.010035044 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964511+0000", 2023-10-09T06:46:29.762 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.3705000000000002e-05 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:37:53.964637+0000", 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000126637 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.763 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218069 8.13 8:cc6ab2b7:::benchmark_data_smithi125_176369_object109001:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 502.53007995299998, 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.396654641, 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218069 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.764 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046569+0000", 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.765 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046567+0000", 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999981 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046574+0000", 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.8180000000000004e-06 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046589+0000", 2023-10-09T06:46:29.766 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4353e-05 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046603+0000", 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.4068e-05 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046869+0000", 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00026645999999999999 2023-10-09T06:46:29.767 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046879+0000", 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.5480000000000007e-06 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.435975+0000", 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3890958170000001 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.768 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.436075+0000", 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00010074800000000001 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.436369+0000", 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029384799999999998 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.769 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.442819+0000", 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0064500520000000004 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.442827+0000", 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.4109999999999999e-06 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443032+0000", 2023-10-09T06:46:29.770 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00020510299999999999 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443070+0000", 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8039e-05 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443100+0000", 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0386999999999999e-05 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.771 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443133+0000", 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.2907999999999997e-05 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443223+0000", 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.0001999999999997e-05 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.772 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218072 8.13 8:c8fab208:::benchmark_data_smithi125_176369_object109004:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 502.53003776100002, 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3985338650000001, 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.773 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218072 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046611+0000", 2023-10-09T06:46:29.774 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046610+0000", 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046614+0000", 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5619999999999999e-06 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.775 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046621+0000", 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.0040000000000002e-06 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046627+0000", 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 6.3790000000000003e-06 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.776 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046905+0000", 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00027833900000000001 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046913+0000", 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.7540000000000006e-06 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438151+0000", 2023-10-09T06:46:29.777 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3912373849999999 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438298+0000", 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014730900000000001 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.438591+0000", 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00029305599999999998 2023-10-09T06:46:29.778 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443037+0000", 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0044463310000000004 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443262+0000", 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022511100000000001 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.779 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443309+0000", 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.6928999999999997e-05 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443348+0000", 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8962999999999999e-05 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.780 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445023+0000", 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001674966 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445053+0000", 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0052999999999998e-05 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.781 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445145+0000", 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.1274999999999996e-05 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218081 8.13 8:cee58cb1:::benchmark_data_smithi125_176369_object109013:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 502.53000537499997, 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3989159959999999, 2023-10-09T06:46:29.782 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218081 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.783 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046643+0000", 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.784 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046645+0000", 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 2.5780000000000001e-06 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046650+0000", 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.7659999999999998e-06 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.785 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046654+0000", 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.5939999999999998e-06 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046938+0000", 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00028391899999999999 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046942+0000", 2023-10-09T06:46:29.786 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2130000000000002e-06 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.440649+0000", 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.3937074489999999 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.440778+0000", 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00012910299999999999 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.787 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.441027+0000", 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00024819300000000001 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443236+0000", 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.0022094219999999999 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.788 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443381+0000", 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00014532100000000001 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.444892+0000", 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.001511091 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.789 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.444989+0000", 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 9.6335999999999994e-05 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445404+0000", 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00041531899999999999 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445446+0000", 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4.2033000000000002e-05 2023-10-09T06:46:29.790 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445559+0000", 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000113003 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: "description": "osd_op(client.45383.0:218101 8.13 8:cd996c6e:::benchmark_data_smithi125_176369_object109033:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-09T06:46:29.791 INFO:teuthology.orchestra.run.smithi125.stdout: "initiated_at": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "age": 502.52999229099999, 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.399543532, 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "type_data": { 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_info": { 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client": "client.45383", 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "client_addr": "172.21.15.125:0/1585219550", 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "tid": 218101 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: "events": [ 2023-10-09T06:46:29.792 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "initiated", 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "throttled", 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.793 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "header_read", 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046656+0000", 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 4294967295.9999995 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "all_read", 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046658+0000", 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.748e-06 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "dispatched", 2023-10-09T06:46:29.794 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046661+0000", 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.4419999999999998e-06 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "queued_for_pg", 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046664+0000", 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.0259999999999999e-06 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046954+0000", 2023-10-09T06:46:29.795 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000290296 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for peered", 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:07.046958+0000", 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.8399999999999997e-06 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "reached_pg", 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443403+0000", 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 1.396444805 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.796 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "started", 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443479+0000", 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.6097e-05 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "waiting for subops from 2,3,5,6", 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.443659+0000", 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000180042 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.797 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "op_commit", 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445158+0000", 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00149876 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445638+0000", 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00048019799999999999 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.798 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445868+0000", 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.00022962899999999999 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445905+0000", 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 3.6668000000000002e-05 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.445957+0000", 2023-10-09T06:46:29.799 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 5.2611999999999997e-05 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "commit_sent", 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.446029+0000", 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 7.1662999999999995e-05 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: }, 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: { 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "event": "done", 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "time": "2023-10-09T06:38:08.446200+0000", 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: "duration": 0.000170946 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.800 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.801 INFO:teuthology.orchestra.run.smithi125.stdout: } 2023-10-09T06:46:29.801 INFO:teuthology.orchestra.run.smithi125.stdout: ] 2023-10-09T06:46:29.801 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:46:29.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:29.802 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:29.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:29.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:29.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:29.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout:avg 66.75 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 2.77263 -> 2.77263 (expected baseline 7.07549) 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.6 with 62 -> 62 pgs (0.928839 -> 0.928839 * mean) 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.0 with 69 -> 69 pgs (1.03371 -> 1.03371 * mean) 2023-10-09T06:46:29.995 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:46:29.996 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:46:29.996 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 1 2023-10-09T06:46:29.996 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:46:29.996 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0427 2023-10-09T06:46:29.996 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.4273 2023-10-09T06:46:29.998 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:46:30.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:30.090 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:30.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 54 16 69383 69367 80.2786 54.3125 0.0115067 0.0124513 2023-10-09T06:46:30.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:30.231 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:30.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:30.344 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:30 smithi155 bash[59495]: cluster 2023-10-09T06:46:29.411606+0000 mgr.y (mgr.25723) 1257 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:30.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:30.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:30 smithi125 bash[77299]: cluster 2023-10-09T06:46:29.411606+0000 mgr.y (mgr.25723) 1257 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:30.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:30 smithi125 bash[80002]: cluster 2023-10-09T06:46:29.411606+0000 mgr.y (mgr.25723) 1257 : cluster 0 pgmap v1363: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-09T06:46:30.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:30.502 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:30.606 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:30.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:30.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:30.745 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:30.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:30.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:30.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:30.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:31.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:31.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:31.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 55 16 70263 70247 79.8189 55 0.0187112 0.0125225 2023-10-09T06:46:31.162 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:31.171 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:31 smithi155 bash[59495]: audit 2023-10-09T06:46:29.989836+0000 mgr.y (mgr.25723) 1258 : audit 0 from='client.35955 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:31.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:31.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:31.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:31.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:31.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:31.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:31 smithi125 bash[77299]: audit 2023-10-09T06:46:29.989836+0000 mgr.y (mgr.25723) 1258 : audit 0 from='client.35955 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:31.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:31] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.33.4" 2023-10-09T06:46:31.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:31 smithi125 bash[80002]: audit 2023-10-09T06:46:29.989836+0000 mgr.y (mgr.25723) 1258 : audit 0 from='client.35955 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:31.482 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:31.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:31.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:31.644 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:31.747 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:31.774 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:31.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:31.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:31.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:31.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:32.004 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:32.041 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:32.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:32.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 56 16 71047 71031 79.2685 49 0.0159643 0.0126097 2023-10-09T06:46:32.145 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:32.153 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:46:32.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:32.185 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:32 smithi125 bash[77299]: cluster 2023-10-09T06:46:31.412213+0000 mgr.y (mgr.25723) 1259 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T06:46:32.186 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:32 smithi125 bash[80002]: cluster 2023-10-09T06:46:31.412213+0000 mgr.y (mgr.25723) 1259 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T06:46:32.275 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:32.285 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:32 smithi155 bash[59495]: cluster 2023-10-09T06:46:31.412213+0000 mgr.y (mgr.25723) 1259 : cluster 0 pgmap v1364: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 57 MiB/s wr, 917 op/s 2023-10-09T06:46:32.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:32.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:32.418 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:32.443 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:32.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:32.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:32.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.727 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:32.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:32.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:32.851 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:32.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:32.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:32.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:33.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.100 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:46:33.100 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T06:46:33.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:33.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 57 16 71903 71887 78.8163 53.5 0.0150021 0.0126828 2023-10-09T06:46:33.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:33.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:33.346 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.375 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:33.477 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:33.612 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:33.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.769 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:33.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.908 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:33.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:34.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:34.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:34.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 58 16 72768 72752 78.3894 54.0625 0.0181682 0.0127509 2023-10-09T06:46:34.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:34.172 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:34.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:34.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:34.412 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:34.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:34.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:34.548 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:34 smithi155 bash[59495]: cluster 2023-10-09T06:46:33.412723+0000 mgr.y (mgr.25723) 1260 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T06:46:34.548 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:34 smithi155 bash[59495]: audit 2023-10-09T06:46:33.611565+0000 mon.a (mon.0) 1570 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:34.549 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:34 smithi155 bash[59495]: audit 2023-10-09T06:46:33.612782+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:34.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:34.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:34.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:34.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:34.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:34.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:34.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[77299]: cluster 2023-10-09T06:46:33.412723+0000 mgr.y (mgr.25723) 1260 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T06:46:34.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[77299]: audit 2023-10-09T06:46:33.611565+0000 mon.a (mon.0) 1570 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:34.833 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[77299]: audit 2023-10-09T06:46:33.612782+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:34.833 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[80002]: cluster 2023-10-09T06:46:33.412723+0000 mgr.y (mgr.25723) 1260 : cluster 0 pgmap v1365: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-09T06:46:34.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[80002]: audit 2023-10-09T06:46:33.611565+0000 mon.a (mon.0) 1570 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:46:34.834 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:34 smithi125 bash[80002]: audit 2023-10-09T06:46:33.612782+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:34.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:34.951 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:34.984 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:34.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:35.086 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:35.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:35.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:35.141 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 59 16 73626 73610 77.9696 53.625 0.017004 0.0128193 2023-10-09T06:46:35.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:35.260 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:35.365 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:35.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.546 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:35.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:35.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:35.782 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:35.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:35.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.944 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:35.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:36.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:36.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:36.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:46:36.138809+0000 min lat: 0.00130822 max lat: 0.364762 avg lat: 0.0128858 2023-10-09T06:46:36.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:46:36.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 60 16 74490 74474 77.57 54 0.0128789 0.0128858 2023-10-09T06:46:36.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.210 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:36.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:36.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:36.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:36.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:36.617 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:36.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:36.722 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.732 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:36 smithi155 bash[59495]: cluster 2023-10-09T06:46:35.413510+0000 mgr.y (mgr.25723) 1261 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:46:36.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:36.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:36.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:36 smithi125 bash[77299]: cluster 2023-10-09T06:46:35.413510+0000 mgr.y (mgr.25723) 1261 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:46:36.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:36 smithi125 bash[80002]: cluster 2023-10-09T06:46:35.413510+0000 mgr.y (mgr.25723) 1261 : cluster 0 pgmap v1366: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:46:36.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:36.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:37.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:37.137 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:37.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 61 16 75360 75344 77.1896 54.375 0.014173 0.0129496 2023-10-09T06:46:37.171 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:37.274 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:37.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:37.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:37.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:37.545 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:46:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:37.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:37.576 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:46:37.603 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:46:37.631 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:46:37.653 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:37.654 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:37.658 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:46:37.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:37.689 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:46:37.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:37.714 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:46:37.749 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:46:37.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.752 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:46:37.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:37.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:37.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:37.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:37.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:37.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:38.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:38.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:38.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 62 16 76247 76231 76.8387 55.4375 0.0205284 0.0130082 2023-10-09T06:46:38.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.213 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:38.319 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:38.341 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:38.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:38.464 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:38.567 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.578 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:38 smithi125 bash[77299]: cluster 2023-10-09T06:46:37.413970+0000 mgr.y (mgr.25723) 1262 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:38.578 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:38 smithi125 bash[77299]: audit 2023-10-09T06:46:37.511554+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.125:0/1010009155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:38.579 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:38 smithi125 bash[80002]: cluster 2023-10-09T06:46:37.413970+0000 mgr.y (mgr.25723) 1262 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:38.579 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:38 smithi125 bash[80002]: audit 2023-10-09T06:46:37.511554+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.125:0/1010009155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:38.593 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:38.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:38.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:38.732 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:38 smithi155 bash[59495]: cluster 2023-10-09T06:46:37.413970+0000 mgr.y (mgr.25723) 1262 : cluster 0 pgmap v1367: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:38.732 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:38 smithi155 bash[59495]: audit 2023-10-09T06:46:37.511554+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.125:0/1010009155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:38.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:38.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:38.986 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:38.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:39.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 63 16 77042 77026 76.4077 49.6875 0.0204428 0.0130822 2023-10-09T06:46:39.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:39.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:39.275 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:39.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:39.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:39.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:39.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:39.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:39.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:39.823 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:39.926 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:39.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:39.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:39.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:40.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:40.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:40.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 64 16 77917 77901 76.0682 54.6875 0.0284808 0.0131415 2023-10-09T06:46:40.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:40.241 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:40.345 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:40.368 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:40.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:40.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:40 smithi155 bash[59495]: cluster 2023-10-09T06:46:39.414561+0000 mgr.y (mgr.25723) 1263 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:40.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:40.492 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:40.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:40.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:40.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:40.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:40.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:40.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:40 smithi125 bash[77299]: cluster 2023-10-09T06:46:39.414561+0000 mgr.y (mgr.25723) 1263 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:40.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:40.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:40.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:40.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:40.986 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:41.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:41.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:41.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:40 smithi125 bash[80002]: cluster 2023-10-09T06:46:39.414561+0000 mgr.y (mgr.25723) 1263 : cluster 0 pgmap v1368: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T06:46:41.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 65 16 78761 78745 75.7094 52.75 0.0219742 0.0132026 2023-10-09T06:46:41.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:41.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:41.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:41.380 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:41.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:41.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:41.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:41.517 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:41] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.33.4" 2023-10-09T06:46:41.531 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:41.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:41.655 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:41.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:41.792 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:41.898 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:41.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:41.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:42.024 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:42.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:42.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 66 16 79612 79596 75.3681 53.1875 0.00929545 0.0132628 2023-10-09T06:46:42.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:42.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:42.276 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:42.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:42.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:42.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:42.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:42.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:42.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:42.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:42.672 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:42 smithi155 bash[59495]: cluster 2023-10-09T06:46:41.415418+0000 mgr.y (mgr.25723) 1264 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:46:42.696 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:42.716 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:46:42.756 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:42 smithi125 bash[77299]: cluster 2023-10-09T06:46:41.415418+0000 mgr.y (mgr.25723) 1264 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:46:42.757 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:42 smithi125 bash[80002]: cluster 2023-10-09T06:46:41.415418+0000 mgr.y (mgr.25723) 1264 : cluster 0 pgmap v1369: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:46:42.801 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:42.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:42.936 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:42.961 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:42.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:43.064 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:43.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:43.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 67 16 80442 80426 75.0174 51.875 0.0171822 0.0133245 2023-10-09T06:46:43.218 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:43.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:43.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:43.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:43.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:43.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:43.475 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:43 smithi125 bash[77299]: audit 2023-10-09T06:46:43.302044+0000 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:46:43.475 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:43 smithi125 bash[80002]: audit 2023-10-09T06:46:43.302044+0000 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:46:43.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:43.498 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:46:43.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:43.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:43.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:43 smithi155 bash[59495]: audit 2023-10-09T06:46:43.302044+0000 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:46:43.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:43.761 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:43.864 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:43.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:43.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:43.992 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:44.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:44.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:44.121 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:44.142 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:44.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 68 16 81332 81316 74.7321 55.625 0.0184548 0.0133748 2023-10-09T06:46:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:44.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:44.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:44.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:44.412 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:44.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:44.554 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[77299]: cluster 2023-10-09T06:46:43.415902+0000 mgr.y (mgr.25723) 1265 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 44 MiB/s wr, 708 op/s 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[77299]: cluster 2023-10-09T06:46:43.469773+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[77299]: cluster 2023-10-09T06:46:43.487894+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[77299]: audit 2023-10-09T06:46:43.488531+0000 mon.a (mon.0) 1574 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[77299]: cluster 2023-10-09T06:46:43.488809+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e300: 8 total, 8 up, 4 in 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[80002]: cluster 2023-10-09T06:46:43.415902+0000 mgr.y (mgr.25723) 1265 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 44 MiB/s wr, 708 op/s 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[80002]: cluster 2023-10-09T06:46:43.469773+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[80002]: cluster 2023-10-09T06:46:43.487894+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T06:46:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[80002]: audit 2023-10-09T06:46:43.488531+0000 mon.a (mon.0) 1574 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:46:44.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:44 smithi125 bash[80002]: cluster 2023-10-09T06:46:43.488809+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e300: 8 total, 8 up, 4 in 2023-10-09T06:46:44.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.697 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:44.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:44.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:44.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:44 smithi155 bash[59495]: cluster 2023-10-09T06:46:43.415902+0000 mgr.y (mgr.25723) 1265 : cluster 0 pgmap v1370: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 44 MiB/s wr, 708 op/s 2023-10-09T06:46:44.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:44 smithi155 bash[59495]: cluster 2023-10-09T06:46:43.469773+0000 mon.a (mon.0) 1573 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:46:44.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:44 smithi155 bash[59495]: cluster 2023-10-09T06:46:43.487894+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e300 with expected crc 2023-10-09T06:46:44.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:44 smithi155 bash[59495]: audit 2023-10-09T06:46:43.488531+0000 mon.a (mon.0) 1574 : audit 1 from='client.? 172.21.15.125:0/380140660' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:46:44.807 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:44 smithi155 bash[59495]: cluster 2023-10-09T06:46:43.488809+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e300: 8 total, 8 up, 4 in 2023-10-09T06:46:44.832 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:44.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:44.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:44.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:45.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:45.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:45.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 69 16 82219 82203 74.4524 55.4375 0.0212836 0.0134256 2023-10-09T06:46:45.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:45.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:45.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:45.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:45.362 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:45.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:45.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:45.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:45.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:45.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:45.730 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:45.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:45.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:45.916 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:45.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:46.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:46.061 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:46.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:46.142 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 70 16 83081 83065 74.1584 53.875 0.0154012 0.0134796 2023-10-09T06:46:46.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:46.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:46.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:46.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:46.430 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:46.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:46.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:46.592 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:46 smithi125 bash[77299]: cluster 2023-10-09T06:46:45.416540+0000 mgr.y (mgr.25723) 1266 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:46.592 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:46 smithi125 bash[80002]: cluster 2023-10-09T06:46:45.416540+0000 mgr.y (mgr.25723) 1266 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:46.605 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:46.709 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:46.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:46.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:46.849 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:46 smithi155 bash[59495]: cluster 2023-10-09T06:46:45.416540+0000 mgr.y (mgr.25723) 1266 : cluster 0 pgmap v1372: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:46.874 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:46.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:46.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:47.005 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:47.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:47.108 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:47.129 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:46:47.129 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:46:47.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 71 16 83932 83916 73.8629 53.1875 0.0417371 0.0135318 2023-10-09T06:46:47.145 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:47.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:47.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:47.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:47.413 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:47.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:46:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:47.494 INFO:teuthology.orchestra.run.smithi125.stdout:moved 15 / 267 (5.61798%) 2023-10-09T06:46:47.494 INFO:teuthology.orchestra.run.smithi125.stdout:avg 66.75 2023-10-09T06:46:47.494 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 2.77263 -> 4.46514 (expected baseline 7.07549) 2023-10-09T06:46:47.494 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.6 with 62 -> 69 pgs (0.928839 -> 1.03371 * mean) 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.0 with 69 -> 63 pgs (1.03371 -> 0.94382 * mean) 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:oload 110 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 3 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0509 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.0560 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 weight 1.0000 -> 0.8235 2023-10-09T06:46:47.495 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 weight 1.0000 -> 0.8998 2023-10-09T06:46:47.519 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:47.548 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:47.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:47 smithi125 bash[77299]: audit 2023-10-09T06:46:47.319136+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]: dispatch 2023-10-09T06:46:47.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:47 smithi125 bash[80002]: audit 2023-10-09T06:46:47.319136+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]: dispatch 2023-10-09T06:46:47.651 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:47.673 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:47.777 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:47.802 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:47.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:47 smithi155 bash[59495]: audit 2023-10-09T06:46:47.319136+0000 mon.a (mon.0) 1576 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]: dispatch 2023-10-09T06:46:47.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:47.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:47.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:48.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:48.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 72 16 84382 84366 73.2276 28.125 0.00479774 0.0135494 2023-10-09T06:46:48.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:48.202 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:48.305 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:48.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:48.443 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:48.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:48.598 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:48 smithi155 bash[59495]: audit 2023-10-09T06:46:47.317218+0000 mgr.y (mgr.25723) 1267 : audit 0 from='client.45869 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:48 smithi155 bash[59495]: cluster 2023-10-09T06:46:47.416888+0000 mgr.y (mgr.25723) 1268 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:48 smithi155 bash[59495]: cluster 2023-10-09T06:46:47.486007+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T06:46:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:48 smithi155 bash[59495]: audit 2023-10-09T06:46:47.490392+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]': finished 2023-10-09T06:46:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:48 smithi155 bash[59495]: cluster 2023-10-09T06:46:47.490467+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e301: 8 total, 8 up, 4 in 2023-10-09T06:46:48.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[80002]: audit 2023-10-09T06:46:47.317218+0000 mgr.y (mgr.25723) 1267 : audit 0 from='client.45869 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:48.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[80002]: cluster 2023-10-09T06:46:47.416888+0000 mgr.y (mgr.25723) 1268 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[80002]: cluster 2023-10-09T06:46:47.486007+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[80002]: audit 2023-10-09T06:46:47.490392+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]': finished 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[80002]: cluster 2023-10-09T06:46:47.490467+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e301: 8 total, 8 up, 4 in 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[77299]: audit 2023-10-09T06:46:47.317218+0000 mgr.y (mgr.25723) 1267 : audit 0 from='client.45869 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[77299]: cluster 2023-10-09T06:46:47.416888+0000 mgr.y (mgr.25723) 1268 : cluster 0 pgmap v1373: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[77299]: cluster 2023-10-09T06:46:47.486007+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e301 with expected crc 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[77299]: audit 2023-10-09T06:46:47.490392+0000 mon.a (mon.0) 1577 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58968','4':'53966'}"}]': finished 2023-10-09T06:46:48.710 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:48 smithi125 bash[77299]: cluster 2023-10-09T06:46:47.490467+0000 mon.a (mon.0) 1578 : cluster 0 osdmap e301: 8 total, 8 up, 4 in 2023-10-09T06:46:48.723 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:48.827 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:48.856 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:48.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:48.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:46:48.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:48.997 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:49.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:49.101 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.130 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:49.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 73 16 84382 84366 72.2244 0 - 0.0135494 2023-10-09T06:46:49.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.263 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:49.366 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:49.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:49.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:49.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:49.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:49.667 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:49.707 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T06:46:49.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T06:46:49.770 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:49.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: cluster 2023-10-09T06:46:48.500907+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: cluster 2023-10-09T06:46:48.501037+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e302: 8 total, 8 up, 4 in 2023-10-09T06:46:49.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: cluster 2023-10-09T06:46:48.503254+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: cluster 2023-10-09T06:46:48.503586+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: cluster 2023-10-09T06:46:48.504084+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.781 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[80002]: audit 2023-10-09T06:46:48.607971+0000 mon.a (mon.0) 1580 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:49.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: cluster 2023-10-09T06:46:48.500907+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.781 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: cluster 2023-10-09T06:46:48.501037+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e302: 8 total, 8 up, 4 in 2023-10-09T06:46:49.782 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: cluster 2023-10-09T06:46:48.503254+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.782 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: cluster 2023-10-09T06:46:48.503586+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.782 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: cluster 2023-10-09T06:46:48.504084+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.782 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:49 smithi125 bash[77299]: audit 2023-10-09T06:46:48.607971+0000 mon.a (mon.0) 1580 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:49.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: cluster 2023-10-09T06:46:48.500907+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: cluster 2023-10-09T06:46:48.501037+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e302: 8 total, 8 up, 4 in 2023-10-09T06:46:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: cluster 2023-10-09T06:46:48.503254+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: cluster 2023-10-09T06:46:48.503586+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: cluster 2023-10-09T06:46:48.504084+0000 osd.4 (osd.4) 261 : cluster 3 failed to encode map e302 with expected crc 2023-10-09T06:46:49.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:49 smithi155 bash[59495]: audit 2023-10-09T06:46:48.607971+0000 mon.a (mon.0) 1580 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:49.902 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:49.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:49.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:50.041 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:50.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:50.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 74 16 84842 84826 71.6369 14.375 0.0147431 0.0139539 2023-10-09T06:46:50.179 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:50.204 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:50.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:50.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:50.339 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:50.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:50.369 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:50.445 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:50.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:50.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:50.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:50.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:50.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.417505+0000 mgr.y (mgr.25723) 1269 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 45 MiB/s wr, 719 op/s 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.501581+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.504195+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e303: 8 total, 8 up, 4 in 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.506217+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.506519+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.732 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.511118+0000 osd.6 (osd.6) 264 : cluster 0 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 300'5227 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[77299]: cluster 2023-10-09T06:46:49.756764+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.417505+0000 mgr.y (mgr.25723) 1269 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 45 MiB/s wr, 719 op/s 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.501581+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.504195+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e303: 8 total, 8 up, 4 in 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.506217+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.506519+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.511118+0000 osd.6 (osd.6) 264 : cluster 0 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 300'5227 2023-10-09T06:46:50.733 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:50 smithi125 bash[80002]: cluster 2023-10-09T06:46:49.756764+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.747 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:50.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:50.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:50.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.417505+0000 mgr.y (mgr.25723) 1269 : cluster 0 pgmap v1376: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 45 MiB/s wr, 719 op/s 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.501581+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.504195+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e303: 8 total, 8 up, 4 in 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.506217+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.506519+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.511118+0000 osd.6 (osd.6) 264 : cluster 0 10.a starting backfill to osd.4 from (0'0,0'0] MAX to 300'5227 2023-10-09T06:46:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:50 smithi155 bash[59495]: cluster 2023-10-09T06:46:49.756764+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e303 with expected crc 2023-10-09T06:46:50.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:51.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:51.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:51.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:51.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:51.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 75 16 85702 85686 71.3983 53.75 0.0122845 0.014001 2023-10-09T06:46:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:51.246 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:51.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:51.386 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:51.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:46:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:46:51] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-10-09T06:46:51.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:51.524 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:51.550 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:51.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:51.653 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:51.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:51.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:51.781 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:51.806 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:51.909 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:51.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:51.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:52.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:52.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:52.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 76 15 86546 86531 71.1537 52.8125 0.0194567 0.0140487 2023-10-09T06:46:52.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.196 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:52.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:52.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:52.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:52.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:52.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:52.715 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:52.725 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:52 smithi155 bash[59495]: cluster 2023-10-09T06:46:51.418312+0000 mgr.y (mgr.25723) 1270 : cluster 0 pgmap v1378: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:52.755 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:52.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:52.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:52.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:52 smithi125 bash[77299]: cluster 2023-10-09T06:46:51.418312+0000 mgr.y (mgr.25723) 1270 : cluster 0 pgmap v1378: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:52.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:52 smithi125 bash[80002]: cluster 2023-10-09T06:46:51.418312+0000 mgr.y (mgr.25723) 1270 : cluster 0 pgmap v1378: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:52.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:52.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:53.018 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:53.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:53.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 77 16 87396 87380 70.9187 53.0625 0.0258976 0.0140962 2023-10-09T06:46:53.145 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:53.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:53.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:53.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:53.411 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:53.519 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:53.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:53.565 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:53.567 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:53.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:53.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:53.792 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:53.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:53.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:53.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:53.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:54.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:54.087 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:54.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 78 16 88220 88204 70.6697 51.5 0.013243 0.0141452 2023-10-09T06:46:54.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:54.221 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:54.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:54.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:54.443 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:54.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:54.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:54.520 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:54 smithi155 bash[59495]: cluster 2023-10-09T06:46:53.418739+0000 mgr.y (mgr.25723) 1271 : cluster 0 pgmap v1379: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:54.577 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:54.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:54.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:54 smithi125 bash[77299]: cluster 2023-10-09T06:46:53.418739+0000 mgr.y (mgr.25723) 1271 : cluster 0 pgmap v1379: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:54.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:54 smithi125 bash[80002]: cluster 2023-10-09T06:46:53.418739+0000 mgr.y (mgr.25723) 1271 : cluster 0 pgmap v1379: 89 pgs: 1 activating+remapped, 2 active+clean+remapped, 86 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 28 MiB/s wr, 444 op/s; 10459/253353 objects misplaced (4.128%); 92 B/s, 0 objects/s recovering 2023-10-09T06:46:54.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:54.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:54.858 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:54.880 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:54.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:54.983 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:55.003 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:55.106 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:55.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:55.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 79 16 88964 88948 70.3637 46.5 0.0103717 0.0142054 2023-10-09T06:46:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:55.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.271 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:55.375 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:55.502 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:55.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:55.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:55.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:55.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:55.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:55.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:55.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:55.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:56.031 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:56.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:56.143 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:2023-10-09T06:46:56.140808+0000 min lat: 0.00130822 max lat: 2.10489 avg lat: 0.014256 2023-10-09T06:46:56.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:46:56.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 80 16 89774 89758 70.1169 50.625 0.0113043 0.014256 2023-10-09T06:46:56.169 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:56.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:56.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:56.324 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:56.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:56.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:56.556 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:56.579 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:56.682 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:56.717 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:56.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:56.827 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:56 smithi125 bash[77299]: cluster 2023-10-09T06:46:55.419433+0000 mgr.y (mgr.25723) 1272 : cluster 0 pgmap v1380: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 867 op/s; 10674/266016 objects misplaced (4.013%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:46:56.828 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:56 smithi125 bash[80002]: cluster 2023-10-09T06:46:55.419433+0000 mgr.y (mgr.25723) 1272 : cluster 0 pgmap v1380: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 867 op/s; 10674/266016 objects misplaced (4.013%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:46:56.844 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:56.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:56.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:56 smithi155 bash[59495]: cluster 2023-10-09T06:46:55.419433+0000 mgr.y (mgr.25723) 1272 : cluster 0 pgmap v1380: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 867 op/s; 10674/266016 objects misplaced (4.013%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:46:56.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:56.978 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:56.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:57.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:57.105 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:57.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 81 16 90594 90578 69.8839 51.25 0.0118909 0.0143035 2023-10-09T06:46:57.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:57.228 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:57.331 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:57.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:57.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:57.510 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:46:57.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:46:57.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:57.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:57 smithi125 bash[77299]: cluster 2023-10-09T06:46:57.419838+0000 mgr.y (mgr.25723) 1273 : cluster 0 pgmap v1381: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 48 MiB/s wr, 771 op/s; 10674/266016 objects misplaced (4.013%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:46:57.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:57 smithi125 bash[80002]: cluster 2023-10-09T06:46:57.419838+0000 mgr.y (mgr.25723) 1273 : cluster 0 pgmap v1381: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 48 MiB/s wr, 771 op/s; 10674/266016 objects misplaced (4.013%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:46:57.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:57 smithi155 bash[59495]: cluster 2023-10-09T06:46:57.419838+0000 mgr.y (mgr.25723) 1273 : cluster 0 pgmap v1381: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 48 MiB/s wr, 771 op/s; 10674/266016 objects misplaced (4.013%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:46:57.648 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:46:57.672 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:46:57.696 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:46:57.720 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:46:57.743 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:57.753 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:46:57.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:46:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:46:57.779 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:46:57.809 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:46:57.837 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.838 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:46:57.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:57.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:57.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:58.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:58.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:58.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:58.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 82 16 91422 91406 69.6627 51.75 0.0207941 0.0143492 2023-10-09T06:46:58.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:58.264 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:58.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:58.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:58.436 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:58.538 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:58.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:58.668 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:46:58 smithi125 bash[77299]: audit 2023-10-09T06:46:57.612898+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.125:0/3211164309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:58.669 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:46:58 smithi125 bash[80002]: audit 2023-10-09T06:46:57.612898+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.125:0/3211164309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:58.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:58.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:58.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:58.829 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:46:58 smithi155 bash[59495]: audit 2023-10-09T06:46:57.612898+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.125:0/3211164309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:46:58.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:58.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:58.983 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:46:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:58.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:46:59.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:59.115 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:46:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:46:59.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 83 16 92227 92211 69.4295 50.3125 0.0152191 0.0143973 2023-10-09T06:46:59.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:59.246 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:59.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:46:59.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:59.381 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:46:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:46:59.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:59.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:59.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:59.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:59.640 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:46:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:46:59.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:59.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:46:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:46:59.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:59.900 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:46:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:46:59.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:00.008 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:00.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:00 smithi155 bash[59495]: cluster 2023-10-09T06:46:59.420261+0000 mgr.y (mgr.25723) 1274 : cluster 0 pgmap v1382: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 43 MiB/s wr, 688 op/s; 10674/266016 objects misplaced (4.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:47:00.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 84 16 93017 93001 69.1906 49.375 0.0213368 0.0144442 2023-10-09T06:47:00.149 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:00.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:00.279 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:00.307 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:00.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:00.554 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:00.561 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:00 smithi125 bash[77299]: cluster 2023-10-09T06:46:59.420261+0000 mgr.y (mgr.25723) 1274 : cluster 0 pgmap v1382: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 43 MiB/s wr, 688 op/s; 10674/266016 objects misplaced (4.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:47:00.562 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:00 smithi125 bash[80002]: cluster 2023-10-09T06:46:59.420261+0000 mgr.y (mgr.25723) 1274 : cluster 0 pgmap v1382: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 43 MiB/s wr, 688 op/s; 10674/266016 objects misplaced (4.013%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:47:00.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:00.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:00.680 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.719 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:00.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:00.961 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:00.985 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:00.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:01.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:01.111 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:01.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 85 16 93778 93762 68.9361 47.5625 0.015738 0.0145008 2023-10-09T06:47:01.215 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:01.243 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:01.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:01.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:01.486 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:01] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-10-09T06:47:01.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:01.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:01.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:01.665 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:01.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:01.802 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:01.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:01.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:01.934 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:01.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:02.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:02.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:02.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 86 16 94598 94582 68.7304 51.25 0.0288462 0.0145444 2023-10-09T06:47:02.186 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:02.207 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:02.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:02.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:02.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:02.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:02.478 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:02 smithi125 bash[80002]: cluster 2023-10-09T06:47:01.421089+0000 mgr.y (mgr.25723) 1275 : cluster 0 pgmap v1383: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 913 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:47:02.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:02.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:02.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:02 smithi155 bash[59495]: cluster 2023-10-09T06:47:01.421089+0000 mgr.y (mgr.25723) 1275 : cluster 0 pgmap v1383: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 913 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:47:02.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:02.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:02.850 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:02.859 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:02 smithi125 bash[77299]: cluster 2023-10-09T06:47:01.421089+0000 mgr.y (mgr.25723) 1275 : cluster 0 pgmap v1383: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 913 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:47:02.872 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:02.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:02.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:03.010 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:03.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:03.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:03.141 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:03.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:03.145 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 87 16 95417 95401 68.5287 51.1875 0.00939935 0.0145859 2023-10-09T06:47:03.246 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:03.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:03.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:03.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:03.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:47:03.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:03.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:03.658 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:03.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:03.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:03.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:03.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:03.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:03.946 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:03.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:04.049 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.076 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:04.144 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 88 16 96246 96230 68.3387 51.8125 0.0123381 0.0146272 2023-10-09T06:47:04.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:04.214 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:04.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.354 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:04.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:04.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:04 smithi155 bash[59495]: cluster 2023-10-09T06:47:03.421537+0000 mgr.y (mgr.25723) 1276 : cluster 0 pgmap v1384: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 43 MiB/s wr, 685 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:47:04.481 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:04 smithi155 bash[59495]: audit 2023-10-09T06:47:03.608509+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:04.485 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:04.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.616 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:04.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:04.861 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:04.885 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:04 smithi125 bash[77299]: cluster 2023-10-09T06:47:03.421537+0000 mgr.y (mgr.25723) 1276 : cluster 0 pgmap v1384: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 43 MiB/s wr, 685 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:47:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:04 smithi125 bash[77299]: audit 2023-10-09T06:47:03.608509+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:04 smithi125 bash[80002]: cluster 2023-10-09T06:47:03.421537+0000 mgr.y (mgr.25723) 1276 : cluster 0 pgmap v1384: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 43 MiB/s wr, 685 op/s; 10736/278031 objects misplaced (3.861%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:47:04.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:04 smithi125 bash[80002]: audit 2023-10-09T06:47:03.608509+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:04.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:04.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:05.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:05.119 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:05.145 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 89 16 97069 97053 68.1487 51.4375 0.0124704 0.0146673 2023-10-09T06:47:05.157 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:05.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:05.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:05.398 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:05.424 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:05.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:05.562 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:05.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:05.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:05.688 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:05.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:05.841 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:05.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:05.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:05.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:05.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:06.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:06.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:06.145 INFO:tasks.radosbench.radosbench.0.smithi125.stdout: 90 1 97883 97882 67.9672 51.8125 0.0110211 0.0147087 2023-10-09T06:47:06.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:06.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:06.473 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:06.497 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:06.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:06.600 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:06.608 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:06 smithi155 bash[59495]: cluster 2023-10-09T06:47:05.422664+0000 mgr.y (mgr.25723) 1277 : cluster 0 pgmap v1385: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 10831/290118 objects misplaced (3.733%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:47:06.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:06.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:06.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:06 smithi125 bash[77299]: cluster 2023-10-09T06:47:05.422664+0000 mgr.y (mgr.25723) 1277 : cluster 0 pgmap v1385: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 10831/290118 objects misplaced (3.733%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:47:06.743 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:06 smithi125 bash[80002]: cluster 2023-10-09T06:47:05.422664+0000 mgr.y (mgr.25723) 1277 : cluster 0 pgmap v1385: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 10831/290118 objects misplaced (3.733%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:47:06.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:06.863 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:06.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:06.997 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.007 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:47:07.007 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:47:07.008 INFO:teuthology.orchestra.run.smithi125.stdout: "complaint_time": 30, 2023-10-09T06:47:07.008 INFO:teuthology.orchestra.run.smithi125.stdout: "num_blocked_ops": 0 2023-10-09T06:47:07.008 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:47:07.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:07.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.145 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total time run: 90.0088 2023-10-09T06:47:07.184 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Total writes made: 97883 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Write size: 65536 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Object size: 65536 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Bandwidth (MB/sec): 67.9676 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Bandwidth: 25.405 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max bandwidth (MB/sec): 152.25 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average IOPS: 1087 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev IOPS: 406.48 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max IOPS: 2436 2023-10-09T06:47:07.185 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min IOPS: 0 2023-10-09T06:47:07.186 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Average Latency(s): 0.0147087 2023-10-09T06:47:07.186 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Stddev Latency(s): 0.0275939 2023-10-09T06:47:07.186 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Max latency(s): 2.10489 2023-10-09T06:47:07.186 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Min latency(s): 0.00130822 2023-10-09T06:47:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.187 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:07.187 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:47:07.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.313 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:07.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:07.497 INFO:teuthology.orchestra.run.smithi125.stdout:moved 14 / 267 (5.24345%) 2023-10-09T06:47:07.497 INFO:teuthology.orchestra.run.smithi125.stdout:avg 66.75 2023-10-09T06:47:07.497 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 4.46514 -> 9.45714 (expected baseline 7.07549) 2023-10-09T06:47:07.497 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.4 with 61 -> 57 pgs (0.913858 -> 0.853933 * mean) 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.7 with 72 -> 76 pgs (1.07865 -> 1.13858 * mean) 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:oload 110 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 3 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0588 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.0646 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 weight 0.8235 -> 0.6701 2023-10-09T06:47:07.498 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0 weight 0.8998 -> 0.8052 2023-10-09T06:47:07.502 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:47:07.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[77299]: audit 2023-10-09T06:47:06.523358+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.125:0/443340165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.551 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[77299]: audit 2023-10-09T06:47:06.523816+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.552 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[77299]: audit 2023-10-09T06:47:07.188823+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]: dispatch 2023-10-09T06:47:07.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[80002]: audit 2023-10-09T06:47:06.523358+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.125:0/443340165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[80002]: audit 2023-10-09T06:47:06.523816+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.552 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:07 smithi125 bash[80002]: audit 2023-10-09T06:47:07.188823+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]: dispatch 2023-10-09T06:47:07.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:07.670 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:07.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:07.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:07.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:07.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:07 smithi155 bash[59495]: audit 2023-10-09T06:47:06.523358+0000 mon.c (mon.2) 123 : audit 1 from='client.? 172.21.15.125:0/443340165' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:07 smithi155 bash[59495]: audit 2023-10-09T06:47:06.523816+0000 mon.a (mon.0) 1583 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:47:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:07 smithi155 bash[59495]: audit 2023-10-09T06:47:07.188823+0000 mon.a (mon.0) 1584 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]: dispatch 2023-10-09T06:47:07.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:07.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:08.057 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:08.080 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:08.182 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:08.203 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:08.306 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:08.337 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:08.440 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:08.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:08.572 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:08.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:08.691 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:47:08.701 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:08.713 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: audit 2023-10-09T06:47:07.186814+0000 mgr.y (mgr.25723) 1278 : audit 0 from='client.35991 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: cluster 2023-10-09T06:47:07.423148+0000 mgr.y (mgr.25723) 1279 : cluster 0 pgmap v1386: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 10831/290118 objects misplaced (3.733%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: cluster 2023-10-09T06:47:07.481520+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: cluster 2023-10-09T06:47:07.492592+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: audit 2023-10-09T06:47:07.493231+0000 mon.a (mon.0) 1586 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: audit 2023-10-09T06:47:07.493363+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]': finished 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: cluster 2023-10-09T06:47:07.493441+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e304: 8 total, 8 up, 4 in 2023-10-09T06:47:08.714 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:08 smithi155 bash[59495]: cluster 2023-10-09T06:47:07.494144+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:08.835 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:08.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:08.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: audit 2023-10-09T06:47:07.186814+0000 mgr.y (mgr.25723) 1278 : audit 0 from='client.35991 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: cluster 2023-10-09T06:47:07.423148+0000 mgr.y (mgr.25723) 1279 : cluster 0 pgmap v1386: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 10831/290118 objects misplaced (3.733%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: cluster 2023-10-09T06:47:07.481520+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: cluster 2023-10-09T06:47:07.492592+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: audit 2023-10-09T06:47:07.493231+0000 mon.a (mon.0) 1586 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: audit 2023-10-09T06:47:07.493363+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]': finished 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: cluster 2023-10-09T06:47:07.493441+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e304: 8 total, 8 up, 4 in 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[77299]: cluster 2023-10-09T06:47:07.494144+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: audit 2023-10-09T06:47:07.186814+0000 mgr.y (mgr.25723) 1278 : audit 0 from='client.35991 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: cluster 2023-10-09T06:47:07.423148+0000 mgr.y (mgr.25723) 1279 : cluster 0 pgmap v1386: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 10831/290118 objects misplaced (3.733%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: cluster 2023-10-09T06:47:07.481520+0000 mon.a (mon.0) 1585 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: cluster 2023-10-09T06:47:07.492592+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: audit 2023-10-09T06:47:07.493231+0000 mon.a (mon.0) 1586 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: audit 2023-10-09T06:47:07.493363+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52767','4':'43915'}"}]': finished 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: cluster 2023-10-09T06:47:07.493441+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e304: 8 total, 8 up, 4 in 2023-10-09T06:47:08.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:08 smithi125 bash[80002]: cluster 2023-10-09T06:47:07.494144+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e304 with expected crc 2023-10-09T06:47:08.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:08.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:08.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:09.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:09.226 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.254 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:09.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:09.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:09.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:09.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:09.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:09.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:09.637 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:09 smithi155 bash[59495]: cluster 2023-10-09T06:47:08.496245+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e305: 8 total, 8 up, 4 in 2023-10-09T06:47:09.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:09 smithi155 bash[59495]: cluster 2023-10-09T06:47:08.498417+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:09 smithi155 bash[59495]: cluster 2023-10-09T06:47:08.498439+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:09 smithi155 bash[59495]: cluster 2023-10-09T06:47:08.500385+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.638 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:09 smithi155 bash[59495]: cluster 2023-10-09T06:47:08.611622+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.660 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:09.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:09.766 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[77299]: cluster 2023-10-09T06:47:08.496245+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e305: 8 total, 8 up, 4 in 2023-10-09T06:47:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[77299]: cluster 2023-10-09T06:47:08.498417+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[77299]: cluster 2023-10-09T06:47:08.498439+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[77299]: cluster 2023-10-09T06:47:08.500385+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[77299]: cluster 2023-10-09T06:47:08.611622+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[80002]: cluster 2023-10-09T06:47:08.496245+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e305: 8 total, 8 up, 4 in 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[80002]: cluster 2023-10-09T06:47:08.498417+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[80002]: cluster 2023-10-09T06:47:08.498439+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[80002]: cluster 2023-10-09T06:47:08.500385+0000 osd.4 (osd.4) 262 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:09 smithi125 bash[80002]: cluster 2023-10-09T06:47:08.611622+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e305 with expected crc 2023-10-09T06:47:09.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:09.917 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:09.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:09.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:09.967 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:47:09.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T06:47:10.020 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:10.061 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:10.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:10.194 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:10.298 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:10.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:10.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:10.448 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:10.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:10.571 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:10.597 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.423735+0000 mgr.y (mgr.25723) 1280 : cluster 0 pgmap v1389: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 31 MiB/s wr, 503 op/s; 10831/290118 objects misplaced (3.733%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:47:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.504873+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.504886+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.506467+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e306: 8 total, 8 up, 4 in 2023-10-09T06:47:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.508467+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.508588+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.508725+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.509473+0000 osd.0 (osd.0) 20 : cluster 0 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 303'6120 2023-10-09T06:47:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.513901+0000 osd.6 (osd.6) 268 : cluster 0 10.f starting backfill to osd.7 from (0'0,0'0] MAX to 303'6085 2023-10-09T06:47:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:10 smithi155 bash[59495]: cluster 2023-10-09T06:47:09.519931+0000 osd.4 (osd.4) 264 : cluster 0 10.4 starting backfill to osd.7 from (0'0,0'0] MAX to 303'6180 2023-10-09T06:47:10.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:10.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:10.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.423735+0000 mgr.y (mgr.25723) 1280 : cluster 0 pgmap v1389: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 31 MiB/s wr, 503 op/s; 10831/290118 objects misplaced (3.733%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:47:10.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.504873+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.727 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.504886+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.506467+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e306: 8 total, 8 up, 4 in 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.508467+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.508588+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.508725+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.509473+0000 osd.0 (osd.0) 20 : cluster 0 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 303'6120 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.513901+0000 osd.6 (osd.6) 268 : cluster 0 10.f starting backfill to osd.7 from (0'0,0'0] MAX to 303'6085 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[80002]: cluster 2023-10-09T06:47:09.519931+0000 osd.4 (osd.4) 264 : cluster 0 10.4 starting backfill to osd.7 from (0'0,0'0] MAX to 303'6180 2023-10-09T06:47:10.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.423735+0000 mgr.y (mgr.25723) 1280 : cluster 0 pgmap v1389: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 31 MiB/s wr, 503 op/s; 10831/290118 objects misplaced (3.733%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.504873+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.504886+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.506467+0000 mon.a (mon.0) 1590 : cluster 0 osdmap e306: 8 total, 8 up, 4 in 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.508467+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.508588+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.508725+0000 osd.4 (osd.4) 263 : cluster 3 failed to encode map e306 with expected crc 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.509473+0000 osd.0 (osd.0) 20 : cluster 0 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 303'6120 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.513901+0000 osd.6 (osd.6) 268 : cluster 0 10.f starting backfill to osd.7 from (0'0,0'0] MAX to 303'6085 2023-10-09T06:47:10.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:10 smithi125 bash[77299]: cluster 2023-10-09T06:47:09.519931+0000 osd.4 (osd.4) 264 : cluster 0 10.4 starting backfill to osd.7 from (0'0,0'0] MAX to 303'6180 2023-10-09T06:47:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:10.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:10.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:10.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:10.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:11.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:11.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:11.138 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:11.243 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:11.268 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:11.371 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:11.381 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:11] "GET /metrics HTTP/1.1" 200 36637 "" "Prometheus/2.33.4" 2023-10-09T06:47:11.395 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:11.500 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:11.537 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:11.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:11.643 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.679 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:11.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:11.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:11.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:11.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:11.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:11.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:11.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:47:12.051 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:12.074 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:12.177 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:12.329 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:12.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:12.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:12.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:12.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:12.726 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:12.737 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:12 smithi125 bash[77299]: cluster 2023-10-09T06:47:11.424458+0000 mgr.y (mgr.25723) 1281 : cluster 0 pgmap v1391: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 12 MiB/s wr, 258 op/s; 21744/292320 objects misplaced (7.438%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:47:12.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:12 smithi125 bash[80002]: cluster 2023-10-09T06:47:11.424458+0000 mgr.y (mgr.25723) 1281 : cluster 0 pgmap v1391: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 12 MiB/s wr, 258 op/s; 21744/292320 objects misplaced (7.438%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:47:12.755 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:12.860 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.885 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:12 smithi155 bash[59495]: cluster 2023-10-09T06:47:11.424458+0000 mgr.y (mgr.25723) 1281 : cluster 0 pgmap v1391: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 12 MiB/s wr, 258 op/s; 21744/292320 objects misplaced (7.438%); 14 MiB/s, 222 objects/s recovering 2023-10-09T06:47:12.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:12.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:12.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:13.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:13.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:13.151 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:13.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:13.278 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:13.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:13.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:13.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:13.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:13.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:13.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:13.670 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:13.775 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:13.811 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:13.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:13.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:13.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:14.050 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:14.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:14.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.249 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:14.262 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:14.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:14.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:14.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:14.505 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:14.513 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:14.537 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:14 smithi125 bash[77299]: cluster 2023-10-09T06:47:13.425003+0000 mgr.y (mgr.25723) 1282 : cluster 0 pgmap v1392: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 13 MiB/s wr, 298 op/s; 27339/291984 objects misplaced (9.363%); 17 MiB/s, 278 objects/s recovering 2023-10-09T06:47:14.537 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:14 smithi125 bash[80002]: cluster 2023-10-09T06:47:13.425003+0000 mgr.y (mgr.25723) 1282 : cluster 0 pgmap v1392: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 13 MiB/s wr, 298 op/s; 27339/291984 objects misplaced (9.363%); 17 MiB/s, 278 objects/s recovering 2023-10-09T06:47:14.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.637 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:14.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:14.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:14.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.880 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:14 smithi155 bash[59495]: cluster 2023-10-09T06:47:13.425003+0000 mgr.y (mgr.25723) 1282 : cluster 0 pgmap v1392: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 170 B/s rd, 13 MiB/s wr, 298 op/s; 27339/291984 objects misplaced (9.363%); 17 MiB/s, 278 objects/s recovering 2023-10-09T06:47:14.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:14.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:15.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:15.031 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:15.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:15.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:15.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:15.269 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:15.295 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:15.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:15.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:15.528 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:15.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:15.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:15.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:15.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:15.813 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:15 smithi125 bash[77299]: audit 2023-10-09T06:47:14.857087+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:47:15.814 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:15 smithi125 bash[80002]: audit 2023-10-09T06:47:14.857087+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:47:15.828 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:15 smithi155 bash[59495]: audit 2023-10-09T06:47:14.857087+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:47:15.930 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:15.956 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:15.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:16.058 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:16.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:16.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:16.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:16.310 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:16.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:16.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:16.479 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:16.539 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:16.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:16.614 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:16.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:16.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:16.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:16.853 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:16 smithi125 bash[77299]: cluster 2023-10-09T06:47:15.425826+0000 mgr.y (mgr.25723) 1283 : cluster 0 pgmap v1393: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 129 B/s rd, 9.5 MiB/s wr, 712 op/s; 26040/280395 objects misplaced (9.287%); 16 MiB/s, 255 objects/s recovering 2023-10-09T06:47:16.854 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:16 smithi125 bash[80002]: cluster 2023-10-09T06:47:15.425826+0000 mgr.y (mgr.25723) 1283 : cluster 0 pgmap v1393: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 129 B/s rd, 9.5 MiB/s wr, 712 op/s; 26040/280395 objects misplaced (9.287%); 16 MiB/s, 255 objects/s recovering 2023-10-09T06:47:16.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:16.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:16 smithi155 bash[59495]: cluster 2023-10-09T06:47:15.425826+0000 mgr.y (mgr.25723) 1283 : cluster 0 pgmap v1393: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 129 B/s rd, 9.5 MiB/s wr, 712 op/s; 26040/280395 objects misplaced (9.287%); 16 MiB/s, 255 objects/s recovering 2023-10-09T06:47:16.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:16.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:16.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:17.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:17.121 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:17.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:17.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:17.367 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:17.398 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:17.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:17.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:17.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:17 smithi155 bash[59495]: cluster 2023-10-09T06:47:17.426308+0000 mgr.y (mgr.25723) 1284 : cluster 0 pgmap v1394: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 8.4 MiB/s wr, 632 op/s; 26040/280395 objects misplaced (9.287%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:47:17.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:17.758 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:17.765 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:17 smithi125 bash[77299]: cluster 2023-10-09T06:47:17.426308+0000 mgr.y (mgr.25723) 1284 : cluster 0 pgmap v1394: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 8.4 MiB/s wr, 632 op/s; 26040/280395 objects misplaced (9.287%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:47:17.766 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:17 smithi125 bash[80002]: cluster 2023-10-09T06:47:17.426308+0000 mgr.y (mgr.25723) 1284 : cluster 0 pgmap v1394: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 8.4 MiB/s wr, 632 op/s; 26040/280395 objects misplaced (9.287%); 14 MiB/s, 227 objects/s recovering 2023-10-09T06:47:17.780 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:17.883 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:17.905 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:17.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:18.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:18.028 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:18.133 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:18.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:18.270 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:18.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:18.563 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.587 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:18.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:18.700 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:18 smithi155 bash[59495]: audit 2023-10-09T06:47:18.608588+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.738 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:18.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:18.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:18.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:18.948 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:18 smithi125 bash[77299]: audit 2023-10-09T06:47:18.608588+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:18.949 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:18 smithi125 bash[80002]: audit 2023-10-09T06:47:18.608588+0000 mon.a (mon.0) 1592 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:18.970 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:18.994 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:18.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:18.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:19.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:19.130 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:47:19.158 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:47:19.184 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:47:19.211 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:47:19.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:19.242 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:47:19.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:19.266 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:47:19.308 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:47:19.342 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:47:19.345 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.345 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.345 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.346 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.346 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.346 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.346 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.346 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:47:19.353 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:19.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:19.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.638 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:19.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:19.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:19 smithi125 bash[77299]: audit 2023-10-09T06:47:19.084732+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.125:0/2148587746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:19.750 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:19 smithi125 bash[77299]: cluster 2023-10-09T06:47:19.426836+0000 mgr.y (mgr.25723) 1285 : cluster 0 pgmap v1395: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.5 MiB/s wr, 591 op/s; 25286/279618 objects misplaced (9.043%); 16 MiB/s, 254 objects/s recovering 2023-10-09T06:47:19.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:19 smithi125 bash[80002]: audit 2023-10-09T06:47:19.084732+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.125:0/2148587746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:19.750 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:19 smithi125 bash[80002]: cluster 2023-10-09T06:47:19.426836+0000 mgr.y (mgr.25723) 1285 : cluster 0 pgmap v1395: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.5 MiB/s wr, 591 op/s; 25286/279618 objects misplaced (9.043%); 16 MiB/s, 254 objects/s recovering 2023-10-09T06:47:19.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:19.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:19.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:19.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:19.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:19.996 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:20.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:20.122 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:20.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:19 smithi155 bash[59495]: audit 2023-10-09T06:47:19.084732+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.125:0/2148587746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:20.133 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:19 smithi155 bash[59495]: cluster 2023-10-09T06:47:19.426836+0000 mgr.y (mgr.25723) 1285 : cluster 0 pgmap v1395: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.5 MiB/s wr, 591 op/s; 25286/279618 objects misplaced (9.043%); 16 MiB/s, 254 objects/s recovering 2023-10-09T06:47:20.154 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:20.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:20.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:20.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:20.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:20.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:20.540 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:20.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:20.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:20.779 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:20.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:20.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:20.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:20.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:21.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:21.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:21.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:21.211 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:21.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:21.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:21.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:21.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:21.478 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:21.482 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:21] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.33.4" 2023-10-09T06:47:21.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:21.617 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:21.724 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:21.770 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:21.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:21.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:21.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:22.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:22.188 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:22.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:22.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:22.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:22.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:22 smithi155 bash[59495]: cluster 2023-10-09T06:47:21.427679+0000 mgr.y (mgr.25723) 1286 : cluster 0 pgmap v1396: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 6.3 MiB/s wr, 803 op/s; 24042/268626 objects misplaced (8.950%); 16 MiB/s, 248 objects/s recovering 2023-10-09T06:47:22.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:22 smithi155 bash[59495]: audit 2023-10-09T06:47:22.146699+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.487 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:22 smithi155 bash[59495]: audit 2023-10-09T06:47:22.154415+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:22.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:22.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[77299]: cluster 2023-10-09T06:47:21.427679+0000 mgr.y (mgr.25723) 1286 : cluster 0 pgmap v1396: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 6.3 MiB/s wr, 803 op/s; 24042/268626 objects misplaced (8.950%); 16 MiB/s, 248 objects/s recovering 2023-10-09T06:47:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[77299]: audit 2023-10-09T06:47:22.146699+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[77299]: audit 2023-10-09T06:47:22.154415+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[80002]: cluster 2023-10-09T06:47:21.427679+0000 mgr.y (mgr.25723) 1286 : cluster 0 pgmap v1396: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 6.3 MiB/s wr, 803 op/s; 24042/268626 objects misplaced (8.950%); 16 MiB/s, 248 objects/s recovering 2023-10-09T06:47:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[80002]: audit 2023-10-09T06:47:22.146699+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:22 smithi125 bash[80002]: audit 2023-10-09T06:47:22.154415+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:22.650 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:22.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:22.791 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:22.897 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:22.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:23.039 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:23.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:23.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:23.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:23.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:23.340 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:23.447 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:23.471 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:23.574 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:23.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:23.720 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:23.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:23.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:23.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:23.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:23.990 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:23.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:24.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:24.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:24.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:24.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:24.392 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:24.495 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.541 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:24.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:24 smithi125 bash[77299]: cluster 2023-10-09T06:47:23.428290+0000 mgr.y (mgr.25723) 1287 : cluster 0 pgmap v1397: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 336 KiB/s wr, 687 op/s; 23382/267954 objects misplaced (8.726%); 11 MiB/s, 175 objects/s recovering 2023-10-09T06:47:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:24 smithi125 bash[80002]: cluster 2023-10-09T06:47:23.428290+0000 mgr.y (mgr.25723) 1287 : cluster 0 pgmap v1397: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 336 KiB/s wr, 687 op/s; 23382/267954 objects misplaced (8.726%); 11 MiB/s, 175 objects/s recovering 2023-10-09T06:47:24.649 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:24.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:24.772 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.782 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:24 smithi155 bash[59495]: cluster 2023-10-09T06:47:23.428290+0000 mgr.y (mgr.25723) 1287 : cluster 0 pgmap v1397: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 336 KiB/s wr, 687 op/s; 23382/267954 objects misplaced (8.726%); 11 MiB/s, 175 objects/s recovering 2023-10-09T06:47:24.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:24.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:24.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:24.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:25.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:25.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:25.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:47:25.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:25.207 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:25.312 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.339 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:25.445 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:25.469 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:25.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:25.594 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:25.699 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.724 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:25.829 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.856 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:25.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:25.992 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:25.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:26.095 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:26.120 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:26.223 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:26.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:26.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:26.378 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:26.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:26.497 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:26 smithi155 bash[59495]: cluster 2023-10-09T06:47:25.429180+0000 mgr.y (mgr.25723) 1288 : cluster 0 pgmap v1398: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 22199/257595 objects misplaced (8.618%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:26.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:26.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:26.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:26.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:26.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:26.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:26.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:26.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:26.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:27.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:27.047 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:27.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:26 smithi125 bash[80002]: cluster 2023-10-09T06:47:25.429180+0000 mgr.y (mgr.25723) 1288 : cluster 0 pgmap v1398: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 22199/257595 objects misplaced (8.618%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:27.140 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:26 smithi125 bash[77299]: cluster 2023-10-09T06:47:25.429180+0000 mgr.y (mgr.25723) 1288 : cluster 0 pgmap v1398: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 22199/257595 objects misplaced (8.618%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:27.153 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:27.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:27.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:27.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:27.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:27.450 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:27.553 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:27.576 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:27.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:27.705 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:27.809 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:27.835 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:27.942 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:27.985 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:27.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:28.094 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.118 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:28.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.262 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:28.330 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:47:28.331 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:47:28.365 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:28.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.501 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:28 smithi155 bash[59495]: cluster 2023-10-09T06:47:27.429652+0000 mgr.y (mgr.25723) 1289 : cluster 0 pgmap v1399: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 22199/257595 objects misplaced (8.618%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:47:28.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:28.630 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:28 smithi125 bash[77299]: cluster 2023-10-09T06:47:27.429652+0000 mgr.y (mgr.25723) 1289 : cluster 0 pgmap v1399: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 22199/257595 objects misplaced (8.618%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:47:28.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:28 smithi125 bash[80002]: cluster 2023-10-09T06:47:27.429652+0000 mgr.y (mgr.25723) 1289 : cluster 0 pgmap v1399: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 22199/257595 objects misplaced (8.618%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T06:47:28.658 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:28.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:28.894 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:28.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:28.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:29.023 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:29.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:29.166 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:29.189 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:29.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:29.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:29.427 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:29.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:29.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:29.584 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:29.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:29.690 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:29.716 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:29.820 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:29.848 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:29.954 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:29.981 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:29.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:30.084 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:30.214 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:30.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:moved 11 / 267 (4.11985%) 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:avg 66.75 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 9.45714 -> 13.3908 (expected baseline 7.07549) 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.0 with 55 -> 58 pgs (0.82397 -> 0.868914 * mean) 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.7 with 76 -> 77 pgs (1.13858 -> 1.15356 * mean) 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:47:30.285 INFO:teuthology.orchestra.run.smithi125.stdout:oload 110 2023-10-09T06:47:30.286 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 1 2023-10-09T06:47:30.286 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:47:30.286 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0541 2023-10-09T06:47:30.286 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.0596 2023-10-09T06:47:30.286 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4 weight 0.6701 -> 0.5418 2023-10-09T06:47:30.288 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:47:30.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:30.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[77299]: cluster 2023-10-09T06:47:29.430140+0000 mgr.y (mgr.25723) 1290 : cluster 0 pgmap v1400: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 21591/256899 objects misplaced (8.404%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:30.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[77299]: audit 2023-10-09T06:47:29.820889+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.364 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[77299]: audit 2023-10-09T06:47:29.831843+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.364 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[73481]: debug 2023-10-09T06:47:30.276+0000 7f8497396700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-10-09T06:47:30.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[80002]: cluster 2023-10-09T06:47:29.430140+0000 mgr.y (mgr.25723) 1290 : cluster 0 pgmap v1400: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 21591/256899 objects misplaced (8.404%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:30.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[80002]: audit 2023-10-09T06:47:29.820889+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.365 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:30 smithi125 bash[80002]: audit 2023-10-09T06:47:29.831843+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:30 smithi155 bash[59495]: cluster 2023-10-09T06:47:29.430140+0000 mgr.y (mgr.25723) 1290 : cluster 0 pgmap v1400: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 21591/256899 objects misplaced (8.404%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:30 smithi155 bash[59495]: audit 2023-10-09T06:47:29.820889+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:30 smithi155 bash[59495]: audit 2023-10-09T06:47:29.831843+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:30.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:30.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:30.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:30.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:30.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:30.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:30.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.888 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:30.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:30.991 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.013 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:31.116 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:31.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[77299]: audit 2023-10-09T06:47:30.280020+0000 mgr.y (mgr.25723) 1291 : audit 0 from='client.55369 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[77299]: audit 2023-10-09T06:47:30.352075+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:47:31.163 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[77299]: audit 2023-10-09T06:47:30.353327+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:47:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[80002]: audit 2023-10-09T06:47:30.280020+0000 mgr.y (mgr.25723) 1291 : audit 0 from='client.55369 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[80002]: audit 2023-10-09T06:47:30.352075+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:47:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[80002]: audit 2023-10-09T06:47:30.353327+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:47:31.164 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[80002]: audit 2023-10-09T06:47:30.360308+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:31.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:31.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:31 smithi155 bash[59495]: audit 2023-10-09T06:47:30.280020+0000 mgr.y (mgr.25723) 1291 : audit 0 from='client.55369 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:31 smithi155 bash[59495]: audit 2023-10-09T06:47:30.352075+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:47:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:31 smithi155 bash[59495]: audit 2023-10-09T06:47:30.353327+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:47:31.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:31 smithi155 bash[59495]: audit 2023-10-09T06:47:30.360308+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:31.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:31.416 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[77299]: audit 2023-10-09T06:47:30.360308+0000 mon.a (mon.0) 1599 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:31.417 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:31] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.33.4" 2023-10-09T06:47:31.430 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:31.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.559 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:31.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:31.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:31.802 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.825 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:31.928 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:31.961 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:31.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:32.066 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:32.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:32.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:32.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:47:32.194 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:32.203 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:32 smithi155 bash[59495]: cluster 2023-10-09T06:47:31.430872+0000 mgr.y (mgr.25723) 1292 : cluster 0 pgmap v1401: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 924 op/s; 20318/246315 objects misplaced (8.249%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:32.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:32.328 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:32.348 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:32.396 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:32 smithi125 bash[80002]: cluster 2023-10-09T06:47:31.430872+0000 mgr.y (mgr.25723) 1292 : cluster 0 pgmap v1401: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 924 op/s; 20318/246315 objects misplaced (8.249%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:32.396 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:32 smithi125 bash[77299]: cluster 2023-10-09T06:47:31.430872+0000 mgr.y (mgr.25723) 1292 : cluster 0 pgmap v1401: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 924 op/s; 20318/246315 objects misplaced (8.249%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:32.451 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.490 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:32.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:32.727 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:32.748 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:32.851 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.894 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:32.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:32.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:33.025 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:33.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:33.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:33.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:33.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:33.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:33.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:33.452 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T06:47:33.452 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T06:47:33.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:33.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:33.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:33.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:33.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:33.833 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:33.933 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:33.937 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:33.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:33.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:34.067 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:34.098 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:34.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:34.244 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:34.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:34.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:34.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:34.513 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:47:34.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:34.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:34 smithi155 bash[59495]: cluster 2023-10-09T06:47:33.438893+0000 mgr.y (mgr.25723) 1293 : cluster 0 pgmap v1402: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 639 op/s; 19681/245577 objects misplaced (8.014%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:34 smithi155 bash[59495]: audit 2023-10-09T06:47:33.608961+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:34 smithi155 bash[59495]: audit 2023-10-09T06:47:33.884928+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.125:0/518696245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:34 smithi155 bash[59495]: audit 2023-10-09T06:47:33.885311+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.645 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:34.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:34.748 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:34.772 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:34.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[77299]: cluster 2023-10-09T06:47:33.438893+0000 mgr.y (mgr.25723) 1293 : cluster 0 pgmap v1402: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 639 op/s; 19681/245577 objects misplaced (8.014%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:34.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[77299]: audit 2023-10-09T06:47:33.608961+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[77299]: audit 2023-10-09T06:47:33.884928+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.125:0/518696245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[77299]: audit 2023-10-09T06:47:33.885311+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[80002]: cluster 2023-10-09T06:47:33.438893+0000 mgr.y (mgr.25723) 1293 : cluster 0 pgmap v1402: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 639 op/s; 19681/245577 objects misplaced (8.014%); 11 MiB/s, 183 objects/s recovering 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[80002]: audit 2023-10-09T06:47:33.608961+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[80002]: audit 2023-10-09T06:47:33.884928+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.125:0/518696245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:34 smithi125 bash[80002]: audit 2023-10-09T06:47:33.885311+0000 mon.a (mon.0) 1601 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:47:34.902 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:34.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:35.005 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:35.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:35.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:35.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:35.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:35.306 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:35.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:35.539 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: cluster 2023-10-09T06:47:34.492581+0000 mon.a (mon.0) 1602 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: cluster 2023-10-09T06:47:34.492642+0000 mon.a (mon.0) 1603 : cluster 1 Cluster is now healthy 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: cluster 2023-10-09T06:47:34.502438+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: cluster 2023-10-09T06:47:34.502538+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: audit 2023-10-09T06:47:34.503235+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[77299]: cluster 2023-10-09T06:47:34.503284+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e307: 8 total, 8 up, 4 in 2023-10-09T06:47:35.548 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: cluster 2023-10-09T06:47:34.492581+0000 mon.a (mon.0) 1602 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:47:35.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: cluster 2023-10-09T06:47:34.492642+0000 mon.a (mon.0) 1603 : cluster 1 Cluster is now healthy 2023-10-09T06:47:35.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: cluster 2023-10-09T06:47:34.502438+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: cluster 2023-10-09T06:47:34.502538+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: audit 2023-10-09T06:47:34.503235+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:47:35.549 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:35 smithi125 bash[80002]: cluster 2023-10-09T06:47:34.503284+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e307: 8 total, 8 up, 4 in 2023-10-09T06:47:35.566 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:35.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:35.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:35.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: cluster 2023-10-09T06:47:34.492581+0000 mon.a (mon.0) 1602 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: cluster 2023-10-09T06:47:34.492642+0000 mon.a (mon.0) 1603 : cluster 1 Cluster is now healthy 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: cluster 2023-10-09T06:47:34.502438+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: cluster 2023-10-09T06:47:34.502538+0000 osd.4 (osd.4) 265 : cluster 3 failed to encode map e307 with expected crc 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: audit 2023-10-09T06:47:34.503235+0000 mon.a (mon.0) 1604 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:47:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:35 smithi155 bash[59495]: cluster 2023-10-09T06:47:34.503284+0000 mon.a (mon.0) 1605 : cluster 0 osdmap e307: 8 total, 8 up, 4 in 2023-10-09T06:47:35.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:35.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:35.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:36.054 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:36.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:36.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:36.190 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:36.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:36.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:36.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:36.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:36.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:36.585 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:36.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:36 smithi155 bash[59495]: cluster 2023-10-09T06:47:35.439744+0000 mgr.y (mgr.25723) 1294 : cluster 0 pgmap v1404: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:36.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:36.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:36.769 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:36.874 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:36.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:36 smithi125 bash[77299]: cluster 2023-10-09T06:47:35.439744+0000 mgr.y (mgr.25723) 1294 : cluster 0 pgmap v1404: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:36.878 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:36 smithi125 bash[80002]: cluster 2023-10-09T06:47:35.439744+0000 mgr.y (mgr.25723) 1294 : cluster 0 pgmap v1404: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:36.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:36.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:36.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:36.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:37.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:37.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:37.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.156 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:37.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:37.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:37.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:37.407 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:37.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:37.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:37.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.691 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:37.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:37.795 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:37.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:37.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:37.944 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:37.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:38.048 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:38.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:38.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:38.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:38.214 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:38.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:38.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:38.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:38.484 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:38.587 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:38.596 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:38 smithi155 bash[59495]: cluster 2023-10-09T06:47:37.440136+0000 mgr.y (mgr.25723) 1295 : cluster 0 pgmap v1405: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:38.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:38.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:38.725 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:38.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:38.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:38.878 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:47:38.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:38 smithi125 bash[77299]: cluster 2023-10-09T06:47:37.440136+0000 mgr.y (mgr.25723) 1295 : cluster 0 pgmap v1405: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:38 smithi125 bash[80002]: cluster 2023-10-09T06:47:37.440136+0000 mgr.y (mgr.25723) 1295 : cluster 0 pgmap v1405: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 18496/234468 objects misplaced (7.888%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:47:38.906 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:38.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:39.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:39.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:39.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:39.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:39.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:39.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:39.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:39.384 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:39.389 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:39.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:39.441 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:39.544 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:39.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:39.669 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:39.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:39.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:39.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:39.923 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:39.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:39.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:40.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:40.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:40.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:40.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:40.345 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:40.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:40 smithi155 bash[59495]: cluster 2023-10-09T06:47:39.440549+0000 mgr.y (mgr.25723) 1296 : cluster 0 pgmap v1406: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 17889/233655 objects misplaced (7.656%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:40.386 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:40.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:40 smithi125 bash[77299]: cluster 2023-10-09T06:47:39.440549+0000 mgr.y (mgr.25723) 1296 : cluster 0 pgmap v1406: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 17889/233655 objects misplaced (7.656%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:40.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:40 smithi125 bash[80002]: cluster 2023-10-09T06:47:39.440549+0000 mgr.y (mgr.25723) 1296 : cluster 0 pgmap v1406: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 17889/233655 objects misplaced (7.656%); 11 MiB/s, 179 objects/s recovering 2023-10-09T06:47:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:40.492 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:40.515 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:40.619 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:40.661 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:40.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:40.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:40.898 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:40.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:40.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:41.042 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:41.066 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:41.170 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:41.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:41.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:41.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:41.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:41.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:41.576 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:41.586 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:41] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-10-09T06:47:41.603 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:41.706 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:41.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:41.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:41.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:41.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:41.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:41.967 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:41.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:42.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:42.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:42.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:42.282 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:42.385 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:42.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:42.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:42.520 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:42 smithi125 bash[77299]: cluster 2023-10-09T06:47:41.441318+0000 mgr.y (mgr.25723) 1297 : cluster 0 pgmap v1407: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 802 op/s; 16720/222213 objects misplaced (7.524%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:47:42.521 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:42 smithi125 bash[80002]: cluster 2023-10-09T06:47:41.441318+0000 mgr.y (mgr.25723) 1297 : cluster 0 pgmap v1407: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 802 op/s; 16720/222213 objects misplaced (7.524%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:47:42.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:42.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:42.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:42.666 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:42.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:42.783 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:42 smithi155 bash[59495]: cluster 2023-10-09T06:47:41.441318+0000 mgr.y (mgr.25723) 1297 : cluster 0 pgmap v1407: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 802 op/s; 16720/222213 objects misplaced (7.524%); 11 MiB/s, 169 objects/s recovering 2023-10-09T06:47:42.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:42.809 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:47:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:42.834 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:47:42.861 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:47:42.888 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:47:42.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:42.919 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:47:42.938 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:42.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:42.950 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:47:42.984 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:47:43.018 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:47:43.020 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.021 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:47:43.040 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:43.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:43.182 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:43.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:43.319 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:43.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:43.450 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:43.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:43.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:43.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:43 smithi155 bash[59495]: audit 2023-10-09T06:47:42.779837+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.125:0/3729057565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:43.708 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:43.737 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:43.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:43.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:43 smithi125 bash[77299]: audit 2023-10-09T06:47:42.779837+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.125:0/3729057565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:43.853 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:43 smithi125 bash[80002]: audit 2023-10-09T06:47:42.779837+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.125:0/3729057565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:47:43.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:43.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:43.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:43.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:44.102 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:44.128 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:44.232 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:44.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:44.362 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:44.383 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:44.488 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:44.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:44.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:44 smithi125 bash[77299]: cluster 2023-10-09T06:47:43.441749+0000 mgr.y (mgr.25723) 1298 : cluster 0 pgmap v1408: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 16097/221463 objects misplaced (7.268%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:47:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:44 smithi125 bash[80002]: cluster 2023-10-09T06:47:43.441749+0000 mgr.y (mgr.25723) 1298 : cluster 0 pgmap v1408: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 16097/221463 objects misplaced (7.268%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:47:44.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:44.759 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:44.768 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:44 smithi155 bash[59495]: cluster 2023-10-09T06:47:43.441749+0000 mgr.y (mgr.25723) 1298 : cluster 0 pgmap v1408: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.5 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 16097/221463 objects misplaced (7.268%); 11 MiB/s, 174 objects/s recovering 2023-10-09T06:47:44.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:44.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:44.921 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:44.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:45.024 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:45.164 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:45.190 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:45.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.322 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:45.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:45.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:45.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:45.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:45.611 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:45.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.754 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:45.858 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:45.901 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:45.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:46.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:46.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:46.136 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:46.164 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:46.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:46.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:46.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:46.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:46.510 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:46 smithi155 bash[59495]: cluster 2023-10-09T06:47:45.442686+0000 mgr.y (mgr.25723) 1299 : cluster 0 pgmap v1409: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.08k op/s; 14940/210192 objects misplaced (7.108%); 12 MiB/s, 192 objects/s recovering 2023-10-09T06:47:46.529 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:46.556 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:46.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:46.690 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:46.776 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:46.794 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:46.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:46.842 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:46 smithi125 bash[77299]: cluster 2023-10-09T06:47:45.442686+0000 mgr.y (mgr.25723) 1299 : cluster 0 pgmap v1409: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.08k op/s; 14940/210192 objects misplaced (7.108%); 12 MiB/s, 192 objects/s recovering 2023-10-09T06:47:46.842 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:46 smithi125 bash[80002]: cluster 2023-10-09T06:47:45.442686+0000 mgr.y (mgr.25723) 1299 : cluster 0 pgmap v1409: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.08k op/s; 14940/210192 objects misplaced (7.108%); 12 MiB/s, 192 objects/s recovering 2023-10-09T06:47:46.926 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:46.947 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:46.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:47.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:47.079 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:47.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:47.208 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:47.315 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:47.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:47.453 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.488 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:47.591 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:47.730 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:47.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:47.855 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.880 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:47.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:47.984 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:48.105 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:47:48.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:48.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:48.282 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:48.313 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:48.422 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.450 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:48.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:48.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:48 smithi155 bash[59495]: cluster 2023-10-09T06:47:47.443201+0000 mgr.y (mgr.25723) 1300 : cluster 0 pgmap v1410: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 14940/210192 objects misplaced (7.108%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:47:48.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:48 smithi155 bash[59495]: cluster 2023-10-09T06:47:47.516914+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:48 smithi155 bash[59495]: cluster 2023-10-09T06:47:47.517702+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e308: 8 total, 8 up, 4 in 2023-10-09T06:47:48.566 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:48 smithi155 bash[59495]: cluster 2023-10-09T06:47:47.519429+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.567 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:48 smithi155 bash[59495]: cluster 2023-10-09T06:47:47.520512+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.592 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:48.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:48.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:48.848 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[77299]: cluster 2023-10-09T06:47:47.443201+0000 mgr.y (mgr.25723) 1300 : cluster 0 pgmap v1410: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 14940/210192 objects misplaced (7.108%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:47:48.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[77299]: cluster 2023-10-09T06:47:47.516914+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[77299]: cluster 2023-10-09T06:47:47.517702+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e308: 8 total, 8 up, 4 in 2023-10-09T06:47:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[77299]: cluster 2023-10-09T06:47:47.519429+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[77299]: cluster 2023-10-09T06:47:47.520512+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[80002]: cluster 2023-10-09T06:47:47.443201+0000 mgr.y (mgr.25723) 1300 : cluster 0 pgmap v1410: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 14940/210192 objects misplaced (7.108%); 8.9 MiB/s, 142 objects/s recovering 2023-10-09T06:47:48.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[80002]: cluster 2023-10-09T06:47:47.516914+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[80002]: cluster 2023-10-09T06:47:47.517702+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e308: 8 total, 8 up, 4 in 2023-10-09T06:47:48.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[80002]: cluster 2023-10-09T06:47:47.519429+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:48 smithi125 bash[80002]: cluster 2023-10-09T06:47:47.520512+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e308 with expected crc 2023-10-09T06:47:48.890 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:48.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:48.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:48.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:49.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:49.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:49.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.288 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:49.394 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:49.417 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:49.520 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: cluster 2023-10-09T06:47:48.526502+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: cluster 2023-10-09T06:47:48.526554+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: cluster 2023-10-09T06:47:48.527859+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e309: 8 total, 8 up, 4 in 2023-10-09T06:47:49.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: cluster 2023-10-09T06:47:48.529526+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: audit 2023-10-09T06:47:48.614306+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:49.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:49 smithi155 bash[59495]: audit 2023-10-09T06:47:48.615444+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:49.550 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:49.653 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.674 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:49.778 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:49.801 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: cluster 2023-10-09T06:47:48.526502+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: cluster 2023-10-09T06:47:48.526554+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: cluster 2023-10-09T06:47:48.527859+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e309: 8 total, 8 up, 4 in 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: cluster 2023-10-09T06:47:48.529526+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: audit 2023-10-09T06:47:48.614306+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[77299]: audit 2023-10-09T06:47:48.615444+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: cluster 2023-10-09T06:47:48.526502+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.819 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: cluster 2023-10-09T06:47:48.526554+0000 osd.4 (osd.4) 266 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: cluster 2023-10-09T06:47:48.527859+0000 mon.a (mon.0) 1607 : cluster 0 osdmap e309: 8 total, 8 up, 4 in 2023-10-09T06:47:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: cluster 2023-10-09T06:47:48.529526+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e309 with expected crc 2023-10-09T06:47:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: audit 2023-10-09T06:47:48.614306+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:47:49.820 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:49 smithi125 bash[80002]: audit 2023-10-09T06:47:48.615444+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:47:49.904 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:49.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:49.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:50.037 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:50.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:50.085 INFO:teuthology.orchestra.run.smithi125.stdout:{ 2023-10-09T06:47:50.085 INFO:teuthology.orchestra.run.smithi125.stdout: "ops": [], 2023-10-09T06:47:50.085 INFO:teuthology.orchestra.run.smithi125.stdout: "num_ops": 0 2023-10-09T06:47:50.085 INFO:teuthology.orchestra.run.smithi125.stdout:} 2023-10-09T06:47:50.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:50.189 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:50.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.324 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:50.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:50.425 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:50.446 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:50.549 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:50.558 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:50 smithi155 bash[59495]: cluster 2023-10-09T06:47:49.443606+0000 mgr.y (mgr.25723) 1301 : cluster 0 pgmap v1413: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 527 op/s; 14242/209553 objects misplaced (6.796%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:47:50.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:50.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:50.709 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:50.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:50.812 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:50.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:50.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:50 smithi125 bash[77299]: cluster 2023-10-09T06:47:49.443606+0000 mgr.y (mgr.25723) 1301 : cluster 0 pgmap v1413: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 527 op/s; 14242/209553 objects misplaced (6.796%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:47:50.840 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:50 smithi125 bash[80002]: cluster 2023-10-09T06:47:49.443606+0000 mgr.y (mgr.25723) 1301 : cluster 0 pgmap v1413: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 79 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 527 op/s; 14242/209553 objects misplaced (6.796%); 12 MiB/s, 188 objects/s recovering 2023-10-09T06:47:50.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:50.955 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:50.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:51.023 INFO:teuthology.orchestra.run.smithi125.stdout:moved 0 / 267 (0%) 2023-10-09T06:47:51.023 INFO:teuthology.orchestra.run.smithi125.stdout:avg 66.75 2023-10-09T06:47:51.023 INFO:teuthology.orchestra.run.smithi125.stdout:stddev 9.45714 -> 9.45714 (expected baseline 7.07549) 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:min osd.0 with 55 -> 55 pgs (0.82397 -> 0.82397 * mean) 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:max osd.7 with 76 -> 76 pgs (1.13858 -> 1.13858 * mean) 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:oload 1000 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:max_change 1 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:max_change_osds 4 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:average_utilization 0.0465 2023-10-09T06:47:51.024 INFO:teuthology.orchestra.run.smithi125.stdout:overload_utilization 0.4646 2023-10-09T06:47:51.026 INFO:teuthology.orchestra.run.smithi125.stderr:no change 2023-10-09T06:47:51.058 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:51.100 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:51.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:51.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:51.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:51.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:51.334 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:51.371 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:51.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:51.515 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:51.581 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:47:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:47:51] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-10-09T06:47:51.622 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:51.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:51 smithi155 bash[59495]: audit 2023-10-09T06:47:51.018766+0000 mgr.y (mgr.25723) 1302 : audit 0 from='client.45917 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:51.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:51 smithi155 bash[59495]: cluster 2023-10-09T06:47:51.444331+0000 mgr.y (mgr.25723) 1303 : cluster 0 pgmap v1414: 89 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 892 op/s; 13347/200043 objects misplaced (6.672%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:47:51.647 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:51.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:51.780 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:51.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:51 smithi125 bash[77299]: audit 2023-10-09T06:47:51.018766+0000 mgr.y (mgr.25723) 1302 : audit 0 from='client.45917 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:51 smithi125 bash[77299]: cluster 2023-10-09T06:47:51.444331+0000 mgr.y (mgr.25723) 1303 : cluster 0 pgmap v1414: 89 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 892 op/s; 13347/200043 objects misplaced (6.672%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:47:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:51 smithi125 bash[80002]: audit 2023-10-09T06:47:51.018766+0000 mgr.y (mgr.25723) 1302 : audit 0 from='client.45917 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:47:51.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:51 smithi125 bash[80002]: cluster 2023-10-09T06:47:51.444331+0000 mgr.y (mgr.25723) 1303 : cluster 0 pgmap v1414: 89 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 892 op/s; 13347/200043 objects misplaced (6.672%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:47:51.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:51.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:51.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:52.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:52.060 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:52.163 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:52.191 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:52.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:52.294 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:52.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:52.423 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:52.449 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:52.554 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:52.579 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:52.682 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:52.708 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:52.811 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:52.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:52.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:52.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:52.968 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:52.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:53.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:53.097 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:53.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:53.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.365 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:53.469 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:53.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:53.615 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:47:53.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:53.683 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:53.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:53.808 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:53.910 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.945 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:53.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:54.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:54.073 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:54.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:54.176 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:54.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:54.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:54.438 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:54.465 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:54.568 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.576 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:54 smithi155 bash[59495]: cluster 2023-10-09T06:47:53.444815+0000 mgr.y (mgr.25723) 1304 : cluster 0 pgmap v1415: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 449 op/s; 13058/199398 objects misplaced (6.549%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:47:54.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:54.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:54.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:54.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:54.864 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:54 smithi125 bash[77299]: cluster 2023-10-09T06:47:53.444815+0000 mgr.y (mgr.25723) 1304 : cluster 0 pgmap v1415: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 449 op/s; 13058/199398 objects misplaced (6.549%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:47:54.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:54 smithi125 bash[80002]: cluster 2023-10-09T06:47:53.444815+0000 mgr.y (mgr.25723) 1304 : cluster 0 pgmap v1415: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 80 active+clean; 4.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 449 op/s; 13058/199398 objects misplaced (6.549%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T06:47:54.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:54.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:54.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:55.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:55.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:55.147 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:55.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:55.280 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:55.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:55.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:55.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:55.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:55.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:55.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:55.680 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:55.785 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:55.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:55.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:55.918 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:55.954 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:55.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:56.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:56.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:56.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:56.217 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:56.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.343 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:56.446 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:56.476 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:56.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:56.526 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:47:56.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.584 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:56 smithi125 bash[77299]: cluster 2023-10-09T06:47:55.445806+0000 mgr.y (mgr.25723) 1305 : cluster 0 pgmap v1416: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 940 op/s; 11922/187620 objects misplaced (6.354%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:47:56.584 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:56 smithi125 bash[77299]: audit 2023-10-09T06:47:55.981464+0000 mon.a (mon.0) 1610 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:47:56.584 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:56 smithi125 bash[80002]: cluster 2023-10-09T06:47:55.445806+0000 mgr.y (mgr.25723) 1305 : cluster 0 pgmap v1416: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 940 op/s; 11922/187620 objects misplaced (6.354%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:47:56.584 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:56 smithi125 bash[80002]: audit 2023-10-09T06:47:55.981464+0000 mon.a (mon.0) 1610 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:47:56.607 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:56.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:56 smithi155 bash[59495]: cluster 2023-10-09T06:47:55.445806+0000 mgr.y (mgr.25723) 1305 : cluster 0 pgmap v1416: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 940 op/s; 11922/187620 objects misplaced (6.354%); 12 MiB/s, 196 objects/s recovering 2023-10-09T06:47:56.720 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:56 smithi155 bash[59495]: audit 2023-10-09T06:47:55.981464+0000 mon.a (mon.0) 1610 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:47:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.746 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:56.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:56.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.871 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:56.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:56.975 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:57.009 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:57.115 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:57.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:57.272 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:57.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:57.375 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:57.410 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:57.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.537 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:57.641 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:57 smithi155 bash[59495]: cluster 2023-10-09T06:47:56.500728+0000 mon.a (mon.0) 1611 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:57 smithi155 bash[59495]: cluster 2023-10-09T06:47:56.517273+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T06:47:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:57 smithi155 bash[59495]: audit 2023-10-09T06:47:56.518260+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:47:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:57 smithi155 bash[59495]: cluster 2023-10-09T06:47:56.518336+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e310: 8 total, 8 up, 4 in 2023-10-09T06:47:57.664 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:57.768 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.777 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[80002]: cluster 2023-10-09T06:47:56.500728+0000 mon.a (mon.0) 1611 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[80002]: cluster 2023-10-09T06:47:56.517273+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[80002]: audit 2023-10-09T06:47:56.518260+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[80002]: cluster 2023-10-09T06:47:56.518336+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e310: 8 total, 8 up, 4 in 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[77299]: cluster 2023-10-09T06:47:56.500728+0000 mon.a (mon.0) 1611 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[77299]: cluster 2023-10-09T06:47:56.517273+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e310 with expected crc 2023-10-09T06:47:57.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[77299]: audit 2023-10-09T06:47:56.518260+0000 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.125:0/1358929089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:47:57.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:57 smithi125 bash[77299]: cluster 2023-10-09T06:47:56.518336+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e310: 8 total, 8 up, 4 in 2023-10-09T06:47:57.792 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:57.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:57.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:57.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:57.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:58.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:58.059 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:47:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:47:58.136 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:47:58.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:58.206 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:58.311 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:58.334 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:58.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:58.468 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:58.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:58.525 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:47:58 smithi125 bash[77299]: cluster 2023-10-09T06:47:57.446251+0000 mgr.y (mgr.25723) 1306 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 11922/187620 objects misplaced (6.354%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T06:47:58.526 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:47:58 smithi125 bash[80002]: cluster 2023-10-09T06:47:57.446251+0000 mgr.y (mgr.25723) 1306 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 11922/187620 objects misplaced (6.354%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T06:47:58.573 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:58.606 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:58.710 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:58.734 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:58.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:58.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:47:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:58.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:47:58.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:47:58 smithi155 bash[59495]: cluster 2023-10-09T06:47:57.446251+0000 mgr.y (mgr.25723) 1306 : cluster 0 pgmap v1418: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 11922/187620 objects misplaced (6.354%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T06:47:58.924 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 7, 6] out_osds: [5, 1, 2, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:47:58.925 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:47:58.925 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-09T06:47:58.925 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:47:58.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:58.997 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:47:59.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:47:59.102 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:59.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:47:59.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:59.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:59.369 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:59.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:47:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:47:59.499 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:59.539 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:47:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:47:59.643 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:59.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:47:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:47:59.717 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:47:59.772 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:59.798 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:47:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:47:59.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:59.932 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:47:59.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:47:59.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:00.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:00.065 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:00.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:00.178 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:00 smithi155 bash[59495]: cluster 2023-10-09T06:47:59.446769+0000 mgr.y (mgr.25723) 1307 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 756 op/s; 11886/186864 objects misplaced (6.361%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:48:00.194 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:00.216 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:00.300 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:00.311 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:00 smithi125 bash[77299]: cluster 2023-10-09T06:47:59.446769+0000 mgr.y (mgr.25723) 1307 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 756 op/s; 11886/186864 objects misplaced (6.361%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:48:00.311 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:00 smithi125 bash[80002]: cluster 2023-10-09T06:47:59.446769+0000 mgr.y (mgr.25723) 1307 : cluster 0 pgmap v1419: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.8 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 756 op/s; 11886/186864 objects misplaced (6.361%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:48:00.325 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:00.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:00.452 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:00.555 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:00.577 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:00.679 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:00.712 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:00.817 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:00.843 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:00.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:00.980 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:00.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:01.087 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:01.124 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:01.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:01.256 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:01.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:01.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:01.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:01.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:01.492 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:01] "GET /metrics HTTP/1.1" 200 36649 "" "Prometheus/2.33.4" 2023-10-09T06:48:01.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:01.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:01.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:01.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:01.767 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:01.871 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:01.896 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:01.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:02.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:02.126 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:02.256 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:02.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:02.383 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.420 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:02.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:02.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:02.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:02 smithi155 bash[59495]: cluster 2023-10-09T06:48:01.447556+0000 mgr.y (mgr.25723) 1308 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 813 op/s; 10824/175632 objects misplaced (6.163%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:48:02.662 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:02.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:02.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:02.799 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:02 smithi125 bash[77299]: cluster 2023-10-09T06:48:01.447556+0000 mgr.y (mgr.25723) 1308 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 813 op/s; 10824/175632 objects misplaced (6.163%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:48:02.800 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:02 smithi125 bash[80002]: cluster 2023-10-09T06:48:01.447556+0000 mgr.y (mgr.25723) 1308 : cluster 0 pgmap v1420: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 813 op/s; 10824/175632 objects misplaced (6.163%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:48:02.816 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:02.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:02.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:03.045 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:03.075 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:03.183 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:03.206 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:03.309 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:03.333 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:03.436 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:03.457 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:03.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:03.598 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:03.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:03.701 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:03.725 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:03.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:03.855 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:03.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:03.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:03.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:04.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:04.129 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:04.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:04.266 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:04.368 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:04.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:04.496 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:04.522 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:04.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:04.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[80002]: cluster 2023-10-09T06:48:03.447965+0000 mgr.y (mgr.25723) 1309 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 817 op/s; 10777/174858 objects misplaced (6.163%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:48:04.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[80002]: audit 2023-10-09T06:48:03.616198+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:04.630 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[80002]: audit 2023-10-09T06:48:03.617520+0000 mon.a (mon.0) 1615 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:04.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[77299]: cluster 2023-10-09T06:48:03.447965+0000 mgr.y (mgr.25723) 1309 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 817 op/s; 10777/174858 objects misplaced (6.163%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:48:04.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[77299]: audit 2023-10-09T06:48:03.616198+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:04.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:04 smithi125 bash[77299]: audit 2023-10-09T06:48:03.617520+0000 mon.a (mon.0) 1615 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:04.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:04.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:04.790 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:04.893 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:04 smithi155 bash[59495]: cluster 2023-10-09T06:48:03.447965+0000 mgr.y (mgr.25723) 1309 : cluster 0 pgmap v1421: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 817 op/s; 10777/174858 objects misplaced (6.163%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:48:04.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:04 smithi155 bash[59495]: audit 2023-10-09T06:48:03.616198+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:04.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:04 smithi155 bash[59495]: audit 2023-10-09T06:48:03.617520+0000 mon.a (mon.0) 1615 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:04.921 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:04.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:05.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:05.048 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:05.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:05.135 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:48:05.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:05.160 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:48:05.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:05.189 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:48:05.211 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:48:05.234 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:48:05.262 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:48:05.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:05.293 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:48:05.321 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:05.327 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:48:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:05.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.329 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.330 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.330 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.330 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.330 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:48:05.428 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.454 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:05.560 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.569 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:05 smithi155 bash[59495]: audit 2023-10-09T06:48:05.098641+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.125:0/1454088001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:05.613 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:05.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:05 smithi125 bash[77299]: audit 2023-10-09T06:48:05.098641+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.125:0/1454088001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:05.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:05 smithi125 bash[80002]: audit 2023-10-09T06:48:05.098641+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.125:0/1454088001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:05.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:05.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:05.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:05.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:05.998 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:06.021 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:06.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:06.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:06.253 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:06.301 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:06.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:06.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:06.518 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:06 smithi155 bash[59495]: cluster 2023-10-09T06:48:05.448539+0000 mgr.y (mgr.25723) 1310 : cluster 0 pgmap v1422: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 9745/163335 objects misplaced (5.966%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:48:06.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:06.553 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:06.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:06.657 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:06.686 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:06.790 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:06.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:06.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:06.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:06.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:06.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:06.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:07.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:07.051 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:06 smithi125 bash[77299]: cluster 2023-10-09T06:48:05.448539+0000 mgr.y (mgr.25723) 1310 : cluster 0 pgmap v1422: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 9745/163335 objects misplaced (5.966%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:48:07.051 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:06 smithi125 bash[80002]: cluster 2023-10-09T06:48:05.448539+0000 mgr.y (mgr.25723) 1310 : cluster 0 pgmap v1422: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 9745/163335 objects misplaced (5.966%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:48:07.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:07.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:07.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:07.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:07.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:07.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:07.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:07.436 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:07.480 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:07.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:07.622 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:07.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:07.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:07.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:07.888 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:07.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:07.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:07.994 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:08.003 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:08.005 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:08.022 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:08.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:08.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:08.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:08.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:08.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:08.389 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:08.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:08.524 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:08.525 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:08 smithi155 bash[59495]: cluster 2023-10-09T06:48:07.448990+0000 mgr.y (mgr.25723) 1311 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 740 op/s; 9745/163335 objects misplaced (5.966%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:08.565 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:08.670 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:08.800 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:08.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:08.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:08 smithi125 bash[77299]: cluster 2023-10-09T06:48:07.448990+0000 mgr.y (mgr.25723) 1311 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 740 op/s; 9745/163335 objects misplaced (5.966%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:08.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:08 smithi125 bash[80002]: cluster 2023-10-09T06:48:07.448990+0000 mgr.y (mgr.25723) 1311 : cluster 0 pgmap v1423: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 740 op/s; 9745/163335 objects misplaced (5.966%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:08.932 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:08.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:08.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:09.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:09.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:09.224 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:09.265 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:09.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:09.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:09.411 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:09.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:09.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:09.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:09.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:09.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:09.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:09.815 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:09.918 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:09.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:09.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:10.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:10.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:10.083 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:48:10.172 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:10.212 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:10.318 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:10.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:10.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:10 smithi125 bash[80002]: cluster 2023-10-09T06:48:09.449589+0000 mgr.y (mgr.25723) 1312 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9702/162609 objects misplaced (5.966%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:48:10.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:10 smithi125 bash[77299]: cluster 2023-10-09T06:48:09.449589+0000 mgr.y (mgr.25723) 1312 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9702/162609 objects misplaced (5.966%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:48:10.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:10 smithi155 bash[59495]: cluster 2023-10-09T06:48:09.449589+0000 mgr.y (mgr.25723) 1312 : cluster 0 pgmap v1424: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.3 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9702/162609 objects misplaced (5.966%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:48:10.458 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:10.495 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:10.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:10.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:10.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:10.729 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:10.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:10.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:10.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:10.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:10.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:10.961 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:48:10.962 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:48:10.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:11.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:11.128 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:11.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:11.217 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:11] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.33.4" 2023-10-09T06:48:11.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:11.287 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:11.392 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:11.426 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:11.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:11.563 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:11.669 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:11.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:11.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:11.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:11.944 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:11.969 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:11.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:12.072 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:12.095 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:12.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:12.198 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:12.226 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:12.330 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:12.355 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:12.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:12.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:12.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:12.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:12 smithi125 bash[80002]: cluster 2023-10-09T06:48:11.450746+0000 mgr.y (mgr.25723) 1313 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 995 op/s; 8682/151014 objects misplaced (5.749%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:48:12.595 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:12 smithi125 bash[77299]: cluster 2023-10-09T06:48:11.450746+0000 mgr.y (mgr.25723) 1313 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 995 op/s; 8682/151014 objects misplaced (5.749%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:48:12.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:12.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:12.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:12.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:12.850 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:12.858 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:12 smithi155 bash[59495]: cluster 2023-10-09T06:48:11.450746+0000 mgr.y (mgr.25723) 1313 : cluster 0 pgmap v1425: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 995 op/s; 8682/151014 objects misplaced (5.749%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:48:12.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:12.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:12.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:13.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:13.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:13.167 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:13.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:13.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:13.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:48:13.422 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:13.483 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:13.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:13.613 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:13.718 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:13.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:13.845 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:13.868 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:13.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:13.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:14.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:14.112 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:14.137 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:14.242 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:14.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:14.377 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:14.404 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:14.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:14.514 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:14 smithi155 bash[59495]: cluster 2023-10-09T06:48:13.451252+0000 mgr.y (mgr.25723) 1314 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 702 op/s; 8639/150330 objects misplaced (5.747%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:48:14.534 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:14.637 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:14.669 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:14.773 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:14.782 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:14 smithi125 bash[77299]: cluster 2023-10-09T06:48:13.451252+0000 mgr.y (mgr.25723) 1314 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 702 op/s; 8639/150330 objects misplaced (5.747%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:48:14.783 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:14 smithi125 bash[80002]: cluster 2023-10-09T06:48:13.451252+0000 mgr.y (mgr.25723) 1314 : cluster 0 pgmap v1426: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 702 op/s; 8639/150330 objects misplaced (5.747%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:48:14.799 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:14.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:14.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:14.928 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:14.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:15.032 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:15.064 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:15.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:15.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:15.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:15.326 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:15.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:15.449 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:15.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:15.589 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:15.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:15.693 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:15.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:15.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:15.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:15.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:15.968 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:15.998 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:16.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:16.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:16.244 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:16.273 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:16.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:16.399 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:16.503 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:16.527 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:16.630 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:16 smithi125 bash[77299]: cluster 2023-10-09T06:48:15.451910+0000 mgr.y (mgr.25723) 1315 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 979 op/s; 7629/139599 objects misplaced (5.465%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:16.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:16 smithi125 bash[80002]: cluster 2023-10-09T06:48:15.451910+0000 mgr.y (mgr.25723) 1315 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 979 op/s; 7629/139599 objects misplaced (5.465%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:16.648 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:48:16.656 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:48:16.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:16.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:16.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:16.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:16.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:16.888 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:16 smithi155 bash[59495]: cluster 2023-10-09T06:48:15.451910+0000 mgr.y (mgr.25723) 1315 : cluster 0 pgmap v1427: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 979 op/s; 7629/139599 objects misplaced (5.465%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:16.919 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:16.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:16.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:17.027 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:17.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:17.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:17.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:17.427 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.457 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:17.533 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:48:17.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:17.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:17.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:17.709 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:17.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:17.824 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:17 smithi125 bash[77299]: audit 2023-10-09T06:48:16.731209+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:17.824 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:17 smithi125 bash[80002]: audit 2023-10-09T06:48:16.731209+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:17.838 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:17.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:17 smithi155 bash[59495]: audit 2023-10-09T06:48:16.731209+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:17.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:17.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:17.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:17.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:18.077 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:18.116 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:18.219 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:18.245 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:18.277 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1427,"stamp":"2023-10-09T06:48:15.451434+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3052476363,"num_objects":46533,"num_object_clones":0,"num_object_copies":139599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7629,"num_objects_unfound":0,"num_objects_dirty":46533,"num_whiteouts":0,"num_read":6224,"num_read_kb":12660,"num_write":150369,"num_write_kb":6279672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11794,"num_bytes_recovered":757565363,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74497,"ondisk_log_size":74497,"up":261,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":13146284,"kb_used_data":9552652,"kb_used_omap":8,"kb_used_meta":3593207,"kb_avail":361834324,"statfs":{"total":383980142592,"available":370518347776,"internally_reserved":0,"allocated":9781915648,"data_stored":9778948387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":3679444224},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-770244608,"num_objects":-11753,"num_object_clones":0,"num_object_copies":-35259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3148,"num_objects_unfound":0,"num_objects_dirty":-11753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11753,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1054,"num_bytes_recovered":69074944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003678"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":309,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531052+0000","last_change":"2023-10-09T06:45:05.136158+0000","last_active":"2023-10-09T06:47:48.531052+0000","last_peered":"2023-10-09T06:47:48.531052+0000","last_clean":"2023-10-09T06:47:48.531052+0000","last_became_active":"2023-10-09T06:45:05.135763+0000","last_became_peered":"2023-10-09T06:45:05.135763+0000","last_unstale":"2023-10-09T06:47:48.531052+0000","last_undegraded":"2023-10-09T06:47:48.531052+0000","last_fullsized":"2023-10-09T06:47:48.531052+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":290,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531895+0000","last_change":"2023-10-09T06:45:33.529264+0000","last_active":"2023-10-09T06:47:48.531895+0000","last_peered":"2023-10-09T06:47:48.531895+0000","last_clean":"2023-10-09T06:47:48.531895+0000","last_became_active":"2023-10-09T06:45:33.528935+0000","last_became_peered":"2023-10-09T06:45:33.528935+0000","last_unstale":"2023-10-09T06:47:48.531895+0000","last_undegraded":"2023-10-09T06:47:48.531895+0000","last_fullsized":"2023-10-09T06:47:48.531895+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":309,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:47:48.535018+0000","last_change":"2023-10-09T06:46:49.522104+0000","last_active":"2023-10-09T06:47:48.535018+0000","last_peered":"2023-10-09T06:47:48.535018+0000","last_clean":"2023-10-09T06:47:48.535018+0000","last_became_active":"2023-10-09T06:46:49.521162+0000","last_became_peered":"2023-10-09T06:46:49.521162+0000","last_unstale":"2023-10-09T06:47:48.535018+0000","last_undegraded":"2023-10-09T06:47:48.535018+0000","last_fullsized":"2023-10-09T06:47:48.535018+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,4],"avail_no_missing":["7","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":176,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.539029+0000","last_change":"2023-10-09T06:45:33.542604+0000","last_active":"2023-10-09T06:47:48.539029+0000","last_peered":"2023-10-09T06:47:48.539029+0000","last_clean":"2023-10-09T06:47:48.539029+0000","last_became_active":"2023-10-09T06:45:33.541873+0000","last_became_peered":"2023-10-09T06:45:33.541873+0000","last_unstale":"2023-10-09T06:47:48.539029+0000","last_undegraded":"2023-10-09T06:47:48.539029+0000","last_fullsized":"2023-10-09T06:47:48.539029+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":329,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535183+0000","last_change":"2023-10-09T06:46:48.521426+0000","last_active":"2023-10-09T06:47:48.535183+0000","last_peered":"2023-10-09T06:47:48.535183+0000","last_clean":"2023-10-09T06:47:48.535183+0000","last_became_active":"2023-10-09T06:46:48.520939+0000","last_became_peered":"2023-10-09T06:46:48.520939+0000","last_unstale":"2023-10-09T06:47:48.535183+0000","last_undegraded":"2023-10-09T06:47:48.535183+0000","last_fullsized":"2023-10-09T06:47:48.535183+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":74,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534864+0000","last_change":"2023-10-09T06:45:33.532461+0000","last_active":"2023-10-09T06:47:48.534864+0000","last_peered":"2023-10-09T06:47:48.534864+0000","last_clean":"2023-10-09T06:47:48.534864+0000","last_became_active":"2023-10-09T06:45:33.532017+0000","last_became_peered":"2023-10-09T06:45:33.532017+0000","last_unstale":"2023-10-09T06:47:48.534864+0000","last_undegraded":"2023-10-09T06:47:48.534864+0000","last_fullsized":"2023-10-09T06:47:48.534864+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":274,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535041+0000","last_change":"2023-10-09T06:45:33.522075+0000","last_active":"2023-10-09T06:47:48.535041+0000","last_peered":"2023-10-09T06:47:48.535041+0000","last_clean":"2023-10-09T06:47:48.535041+0000","last_became_active":"2023-10-09T06:45:33.521697+0000","last_became_peered":"2023-10-09T06:45:33.521697+0000","last_unstale":"2023-10-09T06:47:48.535041+0000","last_undegraded":"2023-10-09T06:47:48.535041+0000","last_fullsized":"2023-10-09T06:47:48.535041+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":321,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.533004+0000","last_change":"2023-10-09T06:45:33.532746+0000","last_active":"2023-10-09T06:47:48.533004+0000","last_peered":"2023-10-09T06:47:48.533004+0000","last_clean":"2023-10-09T06:47:48.533004+0000","last_became_active":"2023-10-09T06:45:33.532241+0000","last_became_peered":"2023-10-09T06:45:33.532241+0000","last_unstale":"2023-10-09T06:47:48.533004+0000","last_undegraded":"2023-10-09T06:47:48.533004+0000","last_fullsized":"2023-10-09T06:47:48.533004+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538980+0000","last_change":"2023-10-09T06:46:48.527343+0000","last_active":"2023-10-09T06:47:48.538980+0000","last_peered":"2023-10-09T06:47:48.538980+0000","last_clean":"2023-10-09T06:47:48.538980+0000","last_became_active":"2023-10-09T06:46:48.526049+0000","last_became_peered":"2023-10-09T06:46:48.526049+0000","last_unstale":"2023-10-09T06:47:48.538980+0000","last_undegraded":"2023-10-09T06:47:48.538980+0000","last_fullsized":"2023-10-09T06:47:48.538980+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":307,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535514+0000","last_change":"2023-10-09T06:45:33.529016+0000","last_active":"2023-10-09T06:47:48.535514+0000","last_peered":"2023-10-09T06:47:48.535514+0000","last_clean":"2023-10-09T06:47:48.535514+0000","last_became_active":"2023-10-09T06:45:33.528659+0000","last_became_peered":"2023-10-09T06:45:33.528659+0000","last_unstale":"2023-10-09T06:47:48.535514+0000","last_undegraded":"2023-10-09T06:47:48.535514+0000","last_fullsized":"2023-10-09T06:47:48.535514+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":223,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538565+0000","last_change":"2023-10-09T06:45:33.810240+0000","last_active":"2023-10-09T06:47:48.538565+0000","last_peered":"2023-10-09T06:47:48.538565+0000","last_clean":"2023-10-09T06:47:48.538565+0000","last_became_active":"2023-10-09T06:45:33.538067+0000","last_became_peered":"2023-10-09T06:45:33.538067+0000","last_unstale":"2023-10-09T06:47:48.538565+0000","last_undegraded":"2023-10-09T06:47:48.538565+0000","last_fullsized":"2023-10-09T06:47:48.538565+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534230+0000","last_change":"2023-10-09T06:45:05.139216+0000","last_active":"2023-10-09T06:47:48.534230+0000","last_peered":"2023-10-09T06:47:48.534230+0000","last_clean":"2023-10-09T06:47:48.534230+0000","last_became_active":"2023-10-09T06:45:05.138786+0000","last_became_peered":"2023-10-09T06:45:05.138786+0000","last_unstale":"2023-10-09T06:47:48.534230+0000","last_undegraded":"2023-10-09T06:47:48.534230+0000","last_fullsized":"2023-10-09T06:47:48.534230+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":396,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531727+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:47:48.531727+0000","last_peered":"2023-10-09T06:47:48.531727+0000","last_clean":"2023-10-09T06:47:48.531727+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:47:48.531727+0000","last_undegraded":"2023-10-09T06:47:48.531727+0000","last_fullsized":"2023-10-09T06:47:48.531727+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":328,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.533183+0000","last_change":"2023-10-09T06:45:05.135916+0000","last_active":"2023-10-09T06:47:48.533183+0000","last_peered":"2023-10-09T06:47:48.533183+0000","last_clean":"2023-10-09T06:47:48.533183+0000","last_became_active":"2023-10-09T06:45:05.135486+0000","last_became_peered":"2023-10-09T06:45:05.135486+0000","last_unstale":"2023-10-09T06:47:48.533183+0000","last_undegraded":"2023-10-09T06:47:48.533183+0000","last_fullsized":"2023-10-09T06:47:48.533183+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":223,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.537952+0000","last_change":"2023-10-09T06:45:05.231838+0000","last_active":"2023-10-09T06:47:48.537952+0000","last_peered":"2023-10-09T06:47:48.537952+0000","last_clean":"2023-10-09T06:47:48.537952+0000","last_became_active":"2023-10-09T06:45:05.139440+0000","last_became_peered":"2023-10-09T06:45:05.139440+0000","last_unstale":"2023-10-09T06:47:48.537952+0000","last_undegraded":"2023-10-09T06:47:48.537952+0000","last_fullsized":"2023-10-09T06:47:48.537952+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":322,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538025+0000","last_change":"2023-10-09T06:45:33.533944+0000","last_active":"2023-10-09T06:47:48.538025+0000","last_peered":"2023-10-09T06:47:48.538025+0000","last_clean":"2023-10-09T06:47:48.538025+0000","last_became_active":"2023-10-09T06:45:33.533570+0000","last_became_peered":"2023-10-09T06:45:33.533570+0000","last_unstale":"2023-10-09T06:47:48.538025+0000","last_undegraded":"2023-10-09T06:47:48.538025+0000","last_fullsized":"2023-10-09T06:47:48.538025+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535548+0000","last_change":"2023-10-09T06:45:33.542558+0000","last_active":"2023-10-09T06:47:48.535548+0000","last_peered":"2023-10-09T06:47:48.535548+0000","last_clean":"2023-10-09T06:47:48.535548+0000","last_became_active":"2023-10-09T06:45:33.542133+0000","last_became_peered":"2023-10-09T06:45:33.542133+0000","last_unstale":"2023-10-09T06:47:48.535548+0000","last_undegraded":"2023-10-09T06:47:48.535548+0000","last_fullsized":"2023-10-09T06:47:48.535548+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":207,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538551+0000","last_change":"2023-10-09T06:45:33.533983+0000","last_active":"2023-10-09T06:47:48.538551+0000","last_peered":"2023-10-09T06:47:48.538551+0000","last_clean":"2023-10-09T06:47:48.538551+0000","last_became_active":"2023-10-09T06:45:33.533641+0000","last_became_peered":"2023-10-09T06:45:33.533641+0000","last_unstale":"2023-10-09T06:47:48.538551+0000","last_undegraded":"2023-10-09T06:47:48.538551+0000","last_fullsized":"2023-10-09T06:47:48.538551+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":256,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535443+0000","last_change":"2023-10-09T06:45:05.141190+0000","last_active":"2023-10-09T06:47:48.535443+0000","last_peered":"2023-10-09T06:47:48.535443+0000","last_clean":"2023-10-09T06:47:48.535443+0000","last_became_active":"2023-10-09T06:45:05.140792+0000","last_became_peered":"2023-10-09T06:45:05.140792+0000","last_unstale":"2023-10-09T06:47:48.535443+0000","last_undegraded":"2023-10-09T06:47:48.535443+0000","last_fullsized":"2023-10-09T06:47:48.535443+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538858+0000","last_change":"2023-10-09T06:45:33.542855+0000","last_active":"2023-10-09T06:47:48.538858+0000","last_peered":"2023-10-09T06:47:48.538858+0000","last_clean":"2023-10-09T06:47:48.538858+0000","last_became_active":"2023-10-09T06:45:33.542281+0000","last_became_peered":"2023-10-09T06:45:33.542281+0000","last_unstale":"2023-10-09T06:47:48.538858+0000","last_undegraded":"2023-10-09T06:47:48.538858+0000","last_fullsized":"2023-10-09T06:47:48.538858+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":95,"reported_epoch":309,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:47:48.533363+0000","last_change":"2023-10-09T06:47:09.527246+0000","last_active":"2023-10-09T06:47:48.533363+0000","last_peered":"2023-10-09T06:47:48.533363+0000","last_clean":"2023-10-09T06:47:48.533363+0000","last_became_active":"2023-10-09T06:47:09.526327+0000","last_became_peered":"2023-10-09T06:47:09.526327+0000","last_unstale":"2023-10-09T06:47:48.533363+0000","last_undegraded":"2023-10-09T06:47:48.533363+0000","last_fullsized":"2023-10-09T06:47:48.533363+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,0],"avail_no_missing":["6","0","7"],"object_location_counts":[{"shards":"0,6,7","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":335,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538503+0000","last_change":"2023-10-09T06:47:08.516385+0000","last_active":"2023-10-09T06:47:48.538503+0000","last_peered":"2023-10-09T06:47:48.538503+0000","last_clean":"2023-10-09T06:47:48.538503+0000","last_became_active":"2023-10-09T06:47:08.515982+0000","last_became_peered":"2023-10-09T06:47:08.515982+0000","last_unstale":"2023-10-09T06:47:48.538503+0000","last_undegraded":"2023-10-09T06:47:48.538503+0000","last_fullsized":"2023-10-09T06:47:48.538503+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"310'9139","reported_seq":9169,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.864416+0000","last_change":"2023-10-09T06:45:08.165059+0000","last_active":"2023-10-09T06:48:14.864416+0000","last_peered":"2023-10-09T06:48:14.864416+0000","last_clean":"2023-10-09T06:48:14.864416+0000","last_became_active":"2023-10-09T06:45:08.164635+0000","last_became_peered":"2023-10-09T06:45:08.164635+0000","last_unstale":"2023-10-09T06:48:14.864416+0000","last_undegraded":"2023-10-09T06:48:14.864416+0000","last_fullsized":"2023-10-09T06:48:14.864416+0000","mapping_epoch":288,"log_start":"299'4200","ondisk_log_start":"299'4200","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4939,"log_dups_size":0,"ondisk_log_size":4939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188022784,"num_objects":2869,"num_object_clones":0,"num_object_copies":8607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9139,"num_write_kb":384256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":206,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538381+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:47:48.538381+0000","last_peered":"2023-10-09T06:47:48.538381+0000","last_clean":"2023-10-09T06:47:48.538381+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:47:48.538381+0000","last_undegraded":"2023-10-09T06:47:48.538381+0000","last_fullsized":"2023-10-09T06:47:48.538381+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535538+0000","last_change":"2023-10-09T06:42:32.485962+0000","last_active":"2023-10-09T06:47:48.535538+0000","last_peered":"2023-10-09T06:47:48.535538+0000","last_clean":"2023-10-09T06:47:48.535538+0000","last_became_active":"2023-10-09T06:42:32.485115+0000","last_became_peered":"2023-10-09T06:42:32.485115+0000","last_unstale":"2023-10-09T06:47:48.535538+0000","last_undegraded":"2023-10-09T06:47:48.535538+0000","last_fullsized":"2023-10-09T06:47:48.535538+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"310'9322","reported_seq":9351,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.884436+0000","last_change":"2023-10-09T06:45:33.527226+0000","last_active":"2023-10-09T06:48:14.884436+0000","last_peered":"2023-10-09T06:48:14.884436+0000","last_clean":"2023-10-09T06:48:14.884436+0000","last_became_active":"2023-10-09T06:45:33.526873+0000","last_became_peered":"2023-10-09T06:45:33.526873+0000","last_unstale":"2023-10-09T06:48:14.884436+0000","last_undegraded":"2023-10-09T06:48:14.884436+0000","last_fullsized":"2023-10-09T06:48:14.884436+0000","mapping_epoch":292,"log_start":"299'4400","ondisk_log_start":"299'4400","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190578688,"num_objects":2908,"num_object_clones":0,"num_object_copies":8724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9322,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":232,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538808+0000","last_change":"2023-10-09T06:47:08.562056+0000","last_active":"2023-10-09T06:47:48.538808+0000","last_peered":"2023-10-09T06:47:48.538808+0000","last_clean":"2023-10-09T06:47:48.538808+0000","last_became_active":"2023-10-09T06:47:08.516681+0000","last_became_peered":"2023-10-09T06:47:08.516681+0000","last_unstale":"2023-10-09T06:47:48.538808+0000","last_undegraded":"2023-10-09T06:47:48.538808+0000","last_fullsized":"2023-10-09T06:47:48.538808+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":316,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531390+0000","last_change":"2023-10-09T06:45:33.535003+0000","last_active":"2023-10-09T06:47:48.531390+0000","last_peered":"2023-10-09T06:47:48.531390+0000","last_clean":"2023-10-09T06:47:48.531390+0000","last_became_active":"2023-10-09T06:45:33.533458+0000","last_became_peered":"2023-10-09T06:45:33.533458+0000","last_unstale":"2023-10-09T06:47:48.531390+0000","last_undegraded":"2023-10-09T06:47:48.531390+0000","last_fullsized":"2023-10-09T06:47:48.531390+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"310'9417","reported_seq":13305,"reported_epoch":310,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T06:48:14.867778+0000","last_change":"2023-10-09T06:47:48.563826+0000","last_active":"2023-10-09T06:48:14.867778+0000","last_peered":"2023-10-09T06:48:14.867778+0000","last_clean":"2023-10-09T06:47:07.490827+0000","last_became_active":"2023-10-09T06:47:09.625912+0000","last_became_peered":"2023-10-09T06:47:09.625912+0000","last_unstale":"2023-10-09T06:48:14.867778+0000","last_undegraded":"2023-10-09T06:48:14.867778+0000","last_fullsized":"2023-10-09T06:48:14.867778+0000","mapping_epoch":305,"log_start":"299'4400","ondisk_log_start":"299'4400","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189595648,"num_objects":2893,"num_object_clones":0,"num_object_copies":8679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1302,"num_objects_unfound":0,"num_objects_dirty":2893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9417,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":2893}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":284,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538439+0000","last_change":"2023-10-09T06:45:33.895855+0000","last_active":"2023-10-09T06:47:48.538439+0000","last_peered":"2023-10-09T06:47:48.538439+0000","last_clean":"2023-10-09T06:47:48.538439+0000","last_became_active":"2023-10-09T06:45:33.538145+0000","last_became_peered":"2023-10-09T06:45:33.538145+0000","last_unstale":"2023-10-09T06:47:48.538439+0000","last_undegraded":"2023-10-09T06:47:48.538439+0000","last_fullsized":"2023-10-09T06:47:48.538439+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.533346+0000","last_change":"2023-10-09T06:47:08.518537+0000","last_active":"2023-10-09T06:47:48.533346+0000","last_peered":"2023-10-09T06:47:48.533346+0000","last_clean":"2023-10-09T06:47:48.533346+0000","last_became_active":"2023-10-09T06:47:08.518084+0000","last_became_peered":"2023-10-09T06:47:08.518084+0000","last_unstale":"2023-10-09T06:47:48.533346+0000","last_undegraded":"2023-10-09T06:47:48.533346+0000","last_fullsized":"2023-10-09T06:47:48.533346+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"310'9516","reported_seq":9544,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.759155+0000","last_change":"2023-10-09T06:45:33.521942+0000","last_active":"2023-10-09T06:48:14.759155+0000","last_peered":"2023-10-09T06:48:14.759155+0000","last_clean":"2023-10-09T06:48:14.759155+0000","last_became_active":"2023-10-09T06:45:33.521557+0000","last_became_peered":"2023-10-09T06:45:33.521557+0000","last_unstale":"2023-10-09T06:48:14.759155+0000","last_undegraded":"2023-10-09T06:48:14.759155+0000","last_fullsized":"2023-10-09T06:48:14.759155+0000","mapping_epoch":292,"log_start":"303'5555","ondisk_log_start":"303'5555","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":3961,"log_dups_size":0,"ondisk_log_size":3961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185729024,"num_objects":2834,"num_object_clones":0,"num_object_copies":8502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9516,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":27,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531397+0000","last_change":"2023-10-09T06:45:33.529820+0000","last_active":"2023-10-09T06:47:48.531397+0000","last_peered":"2023-10-09T06:47:48.531397+0000","last_clean":"2023-10-09T06:47:48.531397+0000","last_became_active":"2023-10-09T06:45:33.529430+0000","last_became_peered":"2023-10-09T06:45:33.529430+0000","last_unstale":"2023-10-09T06:47:48.531397+0000","last_undegraded":"2023-10-09T06:47:48.531397+0000","last_fullsized":"2023-10-09T06:47:48.531397+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":280,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531241+0000","last_change":"2023-10-09T06:45:33.530387+0000","last_active":"2023-10-09T06:47:48.531241+0000","last_peered":"2023-10-09T06:47:48.531241+0000","last_clean":"2023-10-09T06:47:48.531241+0000","last_became_active":"2023-10-09T06:45:33.530012+0000","last_became_peered":"2023-10-09T06:45:33.530012+0000","last_unstale":"2023-10-09T06:47:48.531241+0000","last_undegraded":"2023-10-09T06:47:48.531241+0000","last_fullsized":"2023-10-09T06:47:48.531241+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.2","version":"310'9298","reported_seq":9337,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.841637+0000","last_change":"2023-10-09T06:45:33.526603+0000","last_active":"2023-10-09T06:48:14.841637+0000","last_peered":"2023-10-09T06:48:14.841637+0000","last_clean":"2023-10-09T06:48:14.841637+0000","last_became_active":"2023-10-09T06:45:33.526223+0000","last_became_peered":"2023-10-09T06:45:33.526223+0000","last_unstale":"2023-10-09T06:48:14.841637+0000","last_undegraded":"2023-10-09T06:48:14.841637+0000","last_fullsized":"2023-10-09T06:48:14.841637+0000","mapping_epoch":292,"log_start":"299'4300","ondisk_log_start":"299'4300","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4998,"log_dups_size":0,"ondisk_log_size":4998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9298,"num_write_kb":387712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":44,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531369+0000","last_change":"2023-10-09T06:45:33.813820+0000","last_active":"2023-10-09T06:47:48.531369+0000","last_peered":"2023-10-09T06:47:48.531369+0000","last_clean":"2023-10-09T06:47:48.531369+0000","last_became_active":"2023-10-09T06:45:33.536995+0000","last_became_peered":"2023-10-09T06:45:33.536995+0000","last_unstale":"2023-10-09T06:47:48.531369+0000","last_undegraded":"2023-10-09T06:47:48.531369+0000","last_fullsized":"2023-10-09T06:47:48.531369+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":319,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531872+0000","last_change":"2023-10-09T06:45:33.534114+0000","last_active":"2023-10-09T06:47:48.531872+0000","last_peered":"2023-10-09T06:47:48.531872+0000","last_clean":"2023-10-09T06:47:48.531872+0000","last_became_active":"2023-10-09T06:45:33.533792+0000","last_became_peered":"2023-10-09T06:45:33.533792+0000","last_unstale":"2023-10-09T06:47:48.531872+0000","last_undegraded":"2023-10-09T06:47:48.531872+0000","last_fullsized":"2023-10-09T06:47:48.531872+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"310'9389","reported_seq":9418,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.868290+0000","last_change":"2023-10-09T06:45:33.537089+0000","last_active":"2023-10-09T06:48:14.868290+0000","last_peered":"2023-10-09T06:48:14.868290+0000","last_clean":"2023-10-09T06:48:14.868290+0000","last_became_active":"2023-10-09T06:45:33.536727+0000","last_became_peered":"2023-10-09T06:45:33.536727+0000","last_unstale":"2023-10-09T06:48:14.868290+0000","last_undegraded":"2023-10-09T06:48:14.868290+0000","last_fullsized":"2023-10-09T06:48:14.868290+0000","mapping_epoch":292,"log_start":"299'4400","ondisk_log_start":"299'4400","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9389,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":27,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535169+0000","last_change":"2023-10-09T06:45:33.544298+0000","last_active":"2023-10-09T06:47:48.535169+0000","last_peered":"2023-10-09T06:47:48.535169+0000","last_clean":"2023-10-09T06:47:48.535169+0000","last_became_active":"2023-10-09T06:45:33.543919+0000","last_became_peered":"2023-10-09T06:45:33.543919+0000","last_unstale":"2023-10-09T06:47:48.535169+0000","last_undegraded":"2023-10-09T06:47:48.535169+0000","last_fullsized":"2023-10-09T06:47:48.535169+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.537933+0000","last_change":"2023-10-09T06:46:48.522230+0000","last_active":"2023-10-09T06:47:48.537933+0000","last_peered":"2023-10-09T06:47:48.537933+0000","last_clean":"2023-10-09T06:47:48.537933+0000","last_became_active":"2023-10-09T06:46:48.521789+0000","last_became_peered":"2023-10-09T06:46:48.521789+0000","last_unstale":"2023-10-09T06:47:48.537933+0000","last_undegraded":"2023-10-09T06:47:48.537933+0000","last_fullsized":"2023-10-09T06:47:48.537933+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.0","version":"310'9280","reported_seq":9318,"reported_epoch":310,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:48:14.654410+0000","last_change":"2023-10-09T06:46:49.554320+0000","last_active":"2023-10-09T06:48:14.654410+0000","last_peered":"2023-10-09T06:48:14.654410+0000","last_clean":"2023-10-09T06:48:14.654410+0000","last_became_active":"2023-10-09T06:46:49.520846+0000","last_became_peered":"2023-10-09T06:46:49.520846+0000","last_unstale":"2023-10-09T06:48:14.654410+0000","last_undegraded":"2023-10-09T06:48:14.654410+0000","last_fullsized":"2023-10-09T06:48:14.654410+0000","mapping_epoch":302,"log_start":"300'5111","ondisk_log_start":"300'5111","created":288,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4169,"log_dups_size":0,"ondisk_log_size":4169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191234048,"num_objects":2918,"num_object_clones":0,"num_object_copies":8754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2918,"num_objects_unfound":0,"num_objects_dirty":2918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9280,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":2918}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":303,"reported_epoch":309,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:47:48.535584+0000","last_change":"2023-10-09T06:47:09.522165+0000","last_active":"2023-10-09T06:47:48.535584+0000","last_peered":"2023-10-09T06:47:48.535584+0000","last_clean":"2023-10-09T06:47:48.535584+0000","last_became_active":"2023-10-09T06:47:09.521630+0000","last_became_peered":"2023-10-09T06:47:09.521630+0000","last_unstale":"2023-10-09T06:47:48.535584+0000","last_undegraded":"2023-10-09T06:47:48.535584+0000","last_fullsized":"2023-10-09T06:47:48.535584+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,4],"avail_no_missing":["7","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":328,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531799+0000","last_change":"2023-10-09T06:45:33.533939+0000","last_active":"2023-10-09T06:47:48.531799+0000","last_peered":"2023-10-09T06:47:48.531799+0000","last_clean":"2023-10-09T06:47:48.531799+0000","last_became_active":"2023-10-09T06:45:33.533600+0000","last_became_peered":"2023-10-09T06:45:33.533600+0000","last_unstale":"2023-10-09T06:47:48.531799+0000","last_undegraded":"2023-10-09T06:47:48.531799+0000","last_fullsized":"2023-10-09T06:47:48.531799+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"310'9492","reported_seq":9522,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.882332+0000","last_change":"2023-10-09T06:45:08.167242+0000","last_active":"2023-10-09T06:48:14.882332+0000","last_peered":"2023-10-09T06:48:14.882332+0000","last_clean":"2023-10-09T06:48:14.882332+0000","last_became_active":"2023-10-09T06:45:08.166903+0000","last_became_peered":"2023-10-09T06:45:08.166903+0000","last_unstale":"2023-10-09T06:48:14.882332+0000","last_undegraded":"2023-10-09T06:48:14.882332+0000","last_fullsized":"2023-10-09T06:48:14.882332+0000","mapping_epoch":288,"log_start":"299'4500","ondisk_log_start":"299'4500","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4992,"log_dups_size":0,"ondisk_log_size":4992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188874752,"num_objects":2882,"num_object_clones":0,"num_object_copies":8646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9492,"num_write_kb":395968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":286,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535342+0000","last_change":"2023-10-09T06:45:33.534264+0000","last_active":"2023-10-09T06:47:48.535342+0000","last_peered":"2023-10-09T06:47:48.535342+0000","last_clean":"2023-10-09T06:47:48.535342+0000","last_became_active":"2023-10-09T06:45:33.533903+0000","last_became_peered":"2023-10-09T06:45:33.533903+0000","last_unstale":"2023-10-09T06:47:48.535342+0000","last_undegraded":"2023-10-09T06:47:48.535342+0000","last_fullsized":"2023-10-09T06:47:48.535342+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":138,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531932+0000","last_change":"2023-10-09T06:45:33.529126+0000","last_active":"2023-10-09T06:47:48.531932+0000","last_peered":"2023-10-09T06:47:48.531932+0000","last_clean":"2023-10-09T06:47:48.531932+0000","last_became_active":"2023-10-09T06:45:33.528749+0000","last_became_peered":"2023-10-09T06:45:33.528749+0000","last_unstale":"2023-10-09T06:47:48.531932+0000","last_undegraded":"2023-10-09T06:47:48.531932+0000","last_fullsized":"2023-10-09T06:47:48.531932+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.c","version":"310'9327","reported_seq":9357,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.736227+0000","last_change":"2023-10-09T06:45:08.175117+0000","last_active":"2023-10-09T06:48:14.736227+0000","last_peered":"2023-10-09T06:48:14.736227+0000","last_clean":"2023-10-09T06:48:14.736227+0000","last_became_active":"2023-10-09T06:45:08.174710+0000","last_became_peered":"2023-10-09T06:45:08.174710+0000","last_unstale":"2023-10-09T06:48:14.736227+0000","last_undegraded":"2023-10-09T06:48:14.736227+0000","last_fullsized":"2023-10-09T06:48:14.736227+0000","mapping_epoch":288,"log_start":"303'5291","ondisk_log_start":"303'5291","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9327,"num_write_kb":390848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'183","reported_seq":1138,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538224+0000","last_change":"2023-10-09T06:45:37.931177+0000","last_active":"2023-10-09T06:47:48.538224+0000","last_peered":"2023-10-09T06:47:48.538224+0000","last_clean":"2023-10-09T06:47:48.538224+0000","last_became_active":"2023-10-09T06:45:37.930357+0000","last_became_peered":"2023-10-09T06:45:37.930357+0000","last_unstale":"2023-10-09T06:47:48.538224+0000","last_undegraded":"2023-10-09T06:47:48.538224+0000","last_fullsized":"2023-10-09T06:47:48.538224+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1393,"num_read_kb":3440,"num_write":163,"num_write_kb":2214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538146+0000","last_change":"2023-10-09T06:45:33.535684+0000","last_active":"2023-10-09T06:47:48.538146+0000","last_peered":"2023-10-09T06:47:48.538146+0000","last_clean":"2023-10-09T06:47:48.538146+0000","last_became_active":"2023-10-09T06:45:33.535320+0000","last_became_peered":"2023-10-09T06:45:33.535320+0000","last_unstale":"2023-10-09T06:47:48.538146+0000","last_undegraded":"2023-10-09T06:47:48.538146+0000","last_fullsized":"2023-10-09T06:47:48.538146+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535282+0000","last_change":"2023-10-09T06:47:08.516009+0000","last_active":"2023-10-09T06:47:48.535282+0000","last_peered":"2023-10-09T06:47:48.535282+0000","last_clean":"2023-10-09T06:47:48.535282+0000","last_became_active":"2023-10-09T06:47:08.515431+0000","last_became_peered":"2023-10-09T06:47:08.515431+0000","last_unstale":"2023-10-09T06:47:48.535282+0000","last_undegraded":"2023-10-09T06:47:48.535282+0000","last_fullsized":"2023-10-09T06:47:48.535282+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.a","version":"310'9314","reported_seq":20199,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.672171+0000","last_change":"2023-10-09T06:47:48.548826+0000","last_active":"2023-10-09T06:48:14.672171+0000","last_peered":"2023-10-09T06:48:14.672171+0000","last_clean":"2023-10-09T06:48:14.672171+0000","last_became_active":"2023-10-09T06:47:48.536064+0000","last_became_peered":"2023-10-09T06:47:48.536064+0000","last_unstale":"2023-10-09T06:48:14.672171+0000","last_undegraded":"2023-10-09T06:48:14.672171+0000","last_fullsized":"2023-10-09T06:48:14.672171+0000","mapping_epoch":308,"log_start":"300'5119","ondisk_log_start":"300'5119","created":288,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4195,"log_dups_size":0,"ondisk_log_size":4195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189661184,"num_objects":2894,"num_object_clones":0,"num_object_copies":8682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":390656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5412,"num_bytes_recovered":354680832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'89","reported_seq":45,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535453+0000","last_change":"2023-10-09T06:45:34.328513+0000","last_active":"2023-10-09T06:47:48.535453+0000","last_peered":"2023-10-09T06:47:48.535453+0000","last_clean":"2023-10-09T06:47:48.535453+0000","last_became_active":"2023-10-09T06:45:33.543602+0000","last_became_peered":"2023-10-09T06:45:33.543602+0000","last_unstale":"2023-10-09T06:47:48.535453+0000","last_undegraded":"2023-10-09T06:47:48.535453+0000","last_fullsized":"2023-10-09T06:47:48.535453+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":883,"num_read_kb":1177,"num_write":74,"num_write_kb":1254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":320,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531634+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:47:48.531634+0000","last_peered":"2023-10-09T06:47:48.531634+0000","last_clean":"2023-10-09T06:47:48.531634+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:47:48.531634+0000","last_undegraded":"2023-10-09T06:47:48.531634+0000","last_fullsized":"2023-10-09T06:47:48.531634+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":181,"reported_epoch":309,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:47:48.533155+0000","last_change":"2023-10-09T06:47:09.526688+0000","last_active":"2023-10-09T06:47:48.533155+0000","last_peered":"2023-10-09T06:47:48.533155+0000","last_clean":"2023-10-09T06:47:48.533155+0000","last_became_active":"2023-10-09T06:47:09.526166+0000","last_became_peered":"2023-10-09T06:47:09.526166+0000","last_unstale":"2023-10-09T06:47:48.533155+0000","last_undegraded":"2023-10-09T06:47:48.533155+0000","last_fullsized":"2023-10-09T06:47:48.533155+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,0],"avail_no_missing":["6","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"310'9292","reported_seq":9331,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.749809+0000","last_change":"2023-10-09T06:45:33.524498+0000","last_active":"2023-10-09T06:48:14.749809+0000","last_peered":"2023-10-09T06:48:14.749809+0000","last_clean":"2023-10-09T06:48:14.749809+0000","last_became_active":"2023-10-09T06:45:33.524141+0000","last_became_peered":"2023-10-09T06:45:33.524141+0000","last_unstale":"2023-10-09T06:48:14.749809+0000","last_undegraded":"2023-10-09T06:48:14.749809+0000","last_fullsized":"2023-10-09T06:48:14.749809+0000","mapping_epoch":292,"log_start":"303'5294","ondisk_log_start":"303'5294","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":3998,"log_dups_size":0,"ondisk_log_size":3998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192675840,"num_objects":2940,"num_object_clones":0,"num_object_copies":8820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9292,"num_write_kb":391424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'104","reported_seq":425,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.530695+0000","last_change":"2023-10-09T06:45:36.693264+0000","last_active":"2023-10-09T06:47:48.530695+0000","last_peered":"2023-10-09T06:47:48.530695+0000","last_clean":"2023-10-09T06:47:48.530695+0000","last_became_active":"2023-10-09T06:45:36.692765+0000","last_became_peered":"2023-10-09T06:45:36.692765+0000","last_unstale":"2023-10-09T06:47:48.530695+0000","last_undegraded":"2023-10-09T06:47:48.530695+0000","last_fullsized":"2023-10-09T06:47:48.530695+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":555,"num_read_kb":576,"num_write":92,"num_write_kb":4228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":269,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.533128+0000","last_change":"2023-10-09T06:45:33.532462+0000","last_active":"2023-10-09T06:47:48.533128+0000","last_peered":"2023-10-09T06:47:48.533128+0000","last_clean":"2023-10-09T06:47:48.533128+0000","last_became_active":"2023-10-09T06:45:33.531726+0000","last_became_peered":"2023-10-09T06:45:33.531726+0000","last_unstale":"2023-10-09T06:47:48.533128+0000","last_undegraded":"2023-10-09T06:47:48.533128+0000","last_fullsized":"2023-10-09T06:47:48.533128+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":172,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538923+0000","last_change":"2023-10-09T06:47:08.516021+0000","last_active":"2023-10-09T06:47:48.538923+0000","last_peered":"2023-10-09T06:47:48.538923+0000","last_clean":"2023-10-09T06:47:48.538923+0000","last_became_active":"2023-10-09T06:47:08.515427+0000","last_became_peered":"2023-10-09T06:47:08.515427+0000","last_unstale":"2023-10-09T06:47:48.538923+0000","last_undegraded":"2023-10-09T06:47:48.538923+0000","last_fullsized":"2023-10-09T06:47:48.538923+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.b","version":"310'9169","reported_seq":17429,"reported_epoch":310,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:48:12.954555+0000","last_change":"2023-10-09T06:47:48.555641+0000","last_active":"2023-10-09T06:48:12.954555+0000","last_peered":"2023-10-09T06:48:12.954555+0000","last_clean":"2023-10-09T06:47:07.485913+0000","last_became_active":"2023-10-09T06:47:09.617540+0000","last_became_peered":"2023-10-09T06:47:09.617540+0000","last_unstale":"2023-10-09T06:48:12.954555+0000","last_undegraded":"2023-10-09T06:48:12.954555+0000","last_fullsized":"2023-10-09T06:48:12.954555+0000","mapping_epoch":305,"log_start":"299'4200","ondisk_log_start":"299'4200","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:19:42.195545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":550,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9169,"num_write_kb":390656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4110,"num_bytes_recovered":269352960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":3039}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'155","reported_seq":536,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538444+0000","last_change":"2023-10-09T06:45:37.930874+0000","last_active":"2023-10-09T06:47:48.538444+0000","last_peered":"2023-10-09T06:47:48.538444+0000","last_clean":"2023-10-09T06:47:48.538444+0000","last_became_active":"2023-10-09T06:45:37.929757+0000","last_became_peered":"2023-10-09T06:45:37.929757+0000","last_unstale":"2023-10-09T06:47:48.538444+0000","last_undegraded":"2023-10-09T06:47:48.538444+0000","last_fullsized":"2023-10-09T06:47:48.538444+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":308,"num_read_kb":1997,"num_write":161,"num_write_kb":1496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":605,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535238+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:47:48.535238+0000","last_peered":"2023-10-09T06:47:48.535238+0000","last_clean":"2023-10-09T06:47:48.535238+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:47:48.535238+0000","last_undegraded":"2023-10-09T06:47:48.535238+0000","last_fullsized":"2023-10-09T06:47:48.535238+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":301,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534238+0000","last_change":"2023-10-09T06:45:33.532290+0000","last_active":"2023-10-09T06:47:48.534238+0000","last_peered":"2023-10-09T06:47:48.534238+0000","last_clean":"2023-10-09T06:47:48.534238+0000","last_became_active":"2023-10-09T06:45:33.531814+0000","last_became_peered":"2023-10-09T06:45:33.531814+0000","last_unstale":"2023-10-09T06:47:48.534238+0000","last_undegraded":"2023-10-09T06:47:48.534238+0000","last_fullsized":"2023-10-09T06:47:48.534238+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531785+0000","last_change":"2023-10-09T06:42:32.498269+0000","last_active":"2023-10-09T06:47:48.531785+0000","last_peered":"2023-10-09T06:47:48.531785+0000","last_clean":"2023-10-09T06:47:48.531785+0000","last_became_active":"2023-10-09T06:42:32.497812+0000","last_became_peered":"2023-10-09T06:42:32.497812+0000","last_unstale":"2023-10-09T06:47:48.531785+0000","last_undegraded":"2023-10-09T06:47:48.531785+0000","last_fullsized":"2023-10-09T06:47:48.531785+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"310'9409","reported_seq":9439,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.780727+0000","last_change":"2023-10-09T06:45:08.185207+0000","last_active":"2023-10-09T06:48:14.780727+0000","last_peered":"2023-10-09T06:48:14.780727+0000","last_clean":"2023-10-09T06:48:14.780727+0000","last_became_active":"2023-10-09T06:45:08.184886+0000","last_became_peered":"2023-10-09T06:45:08.184886+0000","last_unstale":"2023-10-09T06:48:14.780727+0000","last_undegraded":"2023-10-09T06:48:14.780727+0000","last_fullsized":"2023-10-09T06:48:14.780727+0000","mapping_epoch":288,"log_start":"303'5390","ondisk_log_start":"303'5390","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4019,"log_dups_size":0,"ondisk_log_size":4019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198115328,"num_objects":3023,"num_object_clones":0,"num_object_copies":9069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":397824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'84","reported_seq":304,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535475+0000","last_change":"2023-10-09T06:45:34.845363+0000","last_active":"2023-10-09T06:47:48.535475+0000","last_peered":"2023-10-09T06:47:48.535475+0000","last_clean":"2023-10-09T06:47:48.535475+0000","last_became_active":"2023-10-09T06:45:33.527346+0000","last_became_peered":"2023-10-09T06:45:33.527346+0000","last_unstale":"2023-10-09T06:47:48.535475+0000","last_undegraded":"2023-10-09T06:47:48.535475+0000","last_fullsized":"2023-10-09T06:47:48.535475+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":531,"num_read_kb":2035,"num_write":62,"num_write_kb":1337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535170+0000","last_change":"2023-10-09T06:46:48.521968+0000","last_active":"2023-10-09T06:47:48.535170+0000","last_peered":"2023-10-09T06:47:48.535170+0000","last_clean":"2023-10-09T06:47:48.535170+0000","last_became_active":"2023-10-09T06:46:48.521339+0000","last_became_peered":"2023-10-09T06:46:48.521339+0000","last_unstale":"2023-10-09T06:47:48.535170+0000","last_undegraded":"2023-10-09T06:47:48.535170+0000","last_fullsized":"2023-10-09T06:47:48.535170+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":290,"reported_epoch":309,"state":"active+clean+remapped","last_fresh":"2023-10-09T06:47:48.538674+0000","last_change":"2023-10-09T06:47:09.520266+0000","last_active":"2023-10-09T06:47:48.538674+0000","last_peered":"2023-10-09T06:47:48.538674+0000","last_clean":"2023-10-09T06:47:48.538674+0000","last_became_active":"2023-10-09T06:47:09.519518+0000","last_became_peered":"2023-10-09T06:47:09.519518+0000","last_unstale":"2023-10-09T06:47:48.538674+0000","last_undegraded":"2023-10-09T06:47:48.538674+0000","last_fullsized":"2023-10-09T06:47:48.538674+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,0],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.d","version":"310'9438","reported_seq":9469,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.855721+0000","last_change":"2023-10-09T06:45:08.165264+0000","last_active":"2023-10-09T06:48:14.855721+0000","last_peered":"2023-10-09T06:48:14.855721+0000","last_clean":"2023-10-09T06:48:14.855721+0000","last_became_active":"2023-10-09T06:45:08.164837+0000","last_became_peered":"2023-10-09T06:45:08.164837+0000","last_unstale":"2023-10-09T06:48:14.855721+0000","last_undegraded":"2023-10-09T06:48:14.855721+0000","last_fullsized":"2023-10-09T06:48:14.855721+0000","mapping_epoch":288,"log_start":"299'4500","ondisk_log_start":"299'4500","created":288,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4938,"log_dups_size":0,"ondisk_log_size":4938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196935704,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9438,"num_write_kb":398145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"77'89","reported_seq":220,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538262+0000","last_change":"2023-10-09T06:19:00.773204+0000","last_active":"2023-10-09T06:47:48.538262+0000","last_peered":"2023-10-09T06:47:48.538262+0000","last_clean":"2023-10-09T06:47:48.538262+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:47:48.538262+0000","last_undegraded":"2023-10-09T06:47:48.538262+0000","last_fullsized":"2023-10-09T06:47:48.538262+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":89,"log_dups_size":0,"ondisk_log_size":89,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":519,"num_read_kb":1450,"num_write":75,"num_write_kb":1052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":34,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.532197+0000","last_change":"2023-10-09T06:45:05.135956+0000","last_active":"2023-10-09T06:47:48.532197+0000","last_peered":"2023-10-09T06:47:48.532197+0000","last_clean":"2023-10-09T06:47:48.532197+0000","last_became_active":"2023-10-09T06:45:05.135547+0000","last_became_peered":"2023-10-09T06:45:05.135547+0000","last_unstale":"2023-10-09T06:47:48.532197+0000","last_undegraded":"2023-10-09T06:47:48.532197+0000","last_fullsized":"2023-10-09T06:47:48.532197+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":313,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538256+0000","last_change":"2023-10-09T06:46:48.524979+0000","last_active":"2023-10-09T06:47:48.538256+0000","last_peered":"2023-10-09T06:47:48.538256+0000","last_clean":"2023-10-09T06:47:48.538256+0000","last_became_active":"2023-10-09T06:46:48.524616+0000","last_became_peered":"2023-10-09T06:46:48.524616+0000","last_unstale":"2023-10-09T06:47:48.538256+0000","last_undegraded":"2023-10-09T06:47:48.538256+0000","last_fullsized":"2023-10-09T06:47:48.538256+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"310'9182","reported_seq":9221,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T06:48:14.828235+0000","last_change":"2023-10-09T06:45:33.534700+0000","last_active":"2023-10-09T06:48:14.828235+0000","last_peered":"2023-10-09T06:48:14.828235+0000","last_clean":"2023-10-09T06:48:14.828235+0000","last_became_active":"2023-10-09T06:45:33.534319+0000","last_became_peered":"2023-10-09T06:45:33.534319+0000","last_unstale":"2023-10-09T06:48:14.828235+0000","last_undegraded":"2023-10-09T06:48:14.828235+0000","last_fullsized":"2023-10-09T06:48:14.828235+0000","mapping_epoch":292,"log_start":"299'4200","ondisk_log_start":"299'4200","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4982,"log_dups_size":0,"ondisk_log_size":4982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186908672,"num_objects":2852,"num_object_clones":0,"num_object_copies":8556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9182,"num_write_kb":385088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":132,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538810+0000","last_change":"2023-10-09T06:45:33.940589+0000","last_active":"2023-10-09T06:47:48.538810+0000","last_peered":"2023-10-09T06:47:48.538810+0000","last_clean":"2023-10-09T06:47:48.538810+0000","last_became_active":"2023-10-09T06:45:33.541861+0000","last_became_peered":"2023-10-09T06:45:33.541861+0000","last_unstale":"2023-10-09T06:47:48.538810+0000","last_undegraded":"2023-10-09T06:47:48.538810+0000","last_fullsized":"2023-10-09T06:47:48.538810+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'105","reported_seq":642,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.532736+0000","last_change":"2023-10-09T06:45:36.693375+0000","last_active":"2023-10-09T06:47:48.532736+0000","last_peered":"2023-10-09T06:47:48.532736+0000","last_clean":"2023-10-09T06:47:48.532736+0000","last_became_active":"2023-10-09T06:45:36.693034+0000","last_became_peered":"2023-10-09T06:45:36.693034+0000","last_unstale":"2023-10-09T06:47:48.532736+0000","last_undegraded":"2023-10-09T06:47:48.532736+0000","last_fullsized":"2023-10-09T06:47:48.532736+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":420,"num_write":76,"num_write_kb":42,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535280+0000","last_change":"2023-10-09T06:46:48.523068+0000","last_active":"2023-10-09T06:47:48.535280+0000","last_peered":"2023-10-09T06:47:48.535280+0000","last_clean":"2023-10-09T06:47:48.535280+0000","last_became_active":"2023-10-09T06:46:48.522635+0000","last_became_peered":"2023-10-09T06:46:48.522635+0000","last_unstale":"2023-10-09T06:47:48.535280+0000","last_undegraded":"2023-10-09T06:47:48.535280+0000","last_fullsized":"2023-10-09T06:47:48.535280+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"310'9279","reported_seq":9328,"reported_epoch":310,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T06:48:14.694081+0000","last_change":"2023-10-09T06:47:09.632857+0000","last_active":"2023-10-09T06:48:14.694081+0000","last_peered":"2023-10-09T06:48:14.694081+0000","last_clean":"2023-10-09T06:47:07.481537+0000","last_became_active":"2023-10-09T06:47:09.622273+0000","last_became_peered":"2023-10-09T06:47:09.622273+0000","last_unstale":"2023-10-09T06:48:14.694081+0000","last_undegraded":"2023-10-09T06:48:14.694081+0000","last_fullsized":"2023-10-09T06:48:14.694081+0000","mapping_epoch":305,"log_start":"300'5076","ondisk_log_start":"300'5076","created":288,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:45:07.127633+0000","last_clean_scrub_stamp":"2023-10-09T06:45:07.127633+0000","objects_scrubbed":0,"log_size":4203,"log_dups_size":0,"ondisk_log_size":4203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2855,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9279,"num_write_kb":388288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":2855}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"77'167","reported_seq":832,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534774+0000","last_change":"2023-10-09T06:45:36.688317+0000","last_active":"2023-10-09T06:47:48.534774+0000","last_peered":"2023-10-09T06:47:48.534774+0000","last_clean":"2023-10-09T06:47:48.534774+0000","last_became_active":"2023-10-09T06:45:36.687770+0000","last_became_peered":"2023-10-09T06:45:36.687770+0000","last_unstale":"2023-10-09T06:47:48.534774+0000","last_undegraded":"2023-10-09T06:47:48.534774+0000","last_fullsized":"2023-10-09T06:47:48.534774+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1204,"num_read_kb":1064,"num_write":143,"num_write_kb":1091,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":51,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531949+0000","last_change":"2023-10-09T06:44:32.353758+0000","last_active":"2023-10-09T06:47:48.531949+0000","last_peered":"2023-10-09T06:47:48.531949+0000","last_clean":"2023-10-09T06:47:48.531949+0000","last_became_active":"2023-10-09T06:44:32.353240+0000","last_became_peered":"2023-10-09T06:44:32.353240+0000","last_unstale":"2023-10-09T06:47:48.531949+0000","last_undegraded":"2023-10-09T06:47:48.531949+0000","last_fullsized":"2023-10-09T06:47:48.531949+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":304,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531145+0000","last_change":"2023-10-09T06:42:32.476668+0000","last_active":"2023-10-09T06:47:48.531145+0000","last_peered":"2023-10-09T06:47:48.531145+0000","last_clean":"2023-10-09T06:47:48.531145+0000","last_became_active":"2023-10-09T06:42:32.476053+0000","last_became_peered":"2023-10-09T06:42:32.476053+0000","last_unstale":"2023-10-09T06:47:48.531145+0000","last_undegraded":"2023-10-09T06:47:48.531145+0000","last_fullsized":"2023-10-09T06:47:48.531145+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":58,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.533041+0000","last_change":"2023-10-09T06:46:48.527610+0000","last_active":"2023-10-09T06:47:48.533041+0000","last_peered":"2023-10-09T06:47:48.533041+0000","last_clean":"2023-10-09T06:47:48.533041+0000","last_became_active":"2023-10-09T06:46:48.526862+0000","last_became_peered":"2023-10-09T06:46:48.526862+0000","last_unstale":"2023-10-09T06:47:48.533041+0000","last_undegraded":"2023-10-09T06:47:48.533041+0000","last_fullsized":"2023-10-09T06:47:48.533041+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":282,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.532029+0000","last_change":"2023-10-09T06:46:48.523235+0000","last_active":"2023-10-09T06:47:48.532029+0000","last_peered":"2023-10-09T06:47:48.532029+0000","last_clean":"2023-10-09T06:47:48.532029+0000","last_became_active":"2023-10-09T06:46:48.522814+0000","last_became_peered":"2023-10-09T06:46:48.522814+0000","last_unstale":"2023-10-09T06:47:48.532029+0000","last_undegraded":"2023-10-09T06:47:48.532029+0000","last_fullsized":"2023-10-09T06:47:48.532029+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":143,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.532351+0000","last_change":"2023-10-09T06:42:32.593843+0000","last_active":"2023-10-09T06:47:48.532351+0000","last_peered":"2023-10-09T06:47:48.532351+0000","last_clean":"2023-10-09T06:47:48.532351+0000","last_became_active":"2023-10-09T06:42:32.483691+0000","last_became_peered":"2023-10-09T06:42:32.483691+0000","last_unstale":"2023-10-09T06:47:48.532351+0000","last_undegraded":"2023-10-09T06:47:48.532351+0000","last_fullsized":"2023-10-09T06:47:48.532351+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":307,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538349+0000","last_change":"2023-10-09T06:47:08.516022+0000","last_active":"2023-10-09T06:47:48.538349+0000","last_peered":"2023-10-09T06:47:48.538349+0000","last_clean":"2023-10-09T06:47:48.538349+0000","last_became_active":"2023-10-09T06:47:08.515542+0000","last_became_peered":"2023-10-09T06:47:08.515542+0000","last_unstale":"2023-10-09T06:47:48.538349+0000","last_undegraded":"2023-10-09T06:47:48.538349+0000","last_fullsized":"2023-10-09T06:47:48.538349+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":156,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.531687+0000","last_change":"2023-10-09T06:45:33.762363+0000","last_active":"2023-10-09T06:47:48.531687+0000","last_peered":"2023-10-09T06:47:48.531687+0000","last_clean":"2023-10-09T06:47:48.531687+0000","last_became_active":"2023-10-09T06:45:33.530125+0000","last_became_peered":"2023-10-09T06:45:33.530125+0000","last_unstale":"2023-10-09T06:47:48.531687+0000","last_undegraded":"2023-10-09T06:47:48.531687+0000","last_fullsized":"2023-10-09T06:47:48.531687+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534984+0000","last_change":"2023-10-09T06:46:48.521277+0000","last_active":"2023-10-09T06:47:48.534984+0000","last_peered":"2023-10-09T06:47:48.534984+0000","last_clean":"2023-10-09T06:47:48.534984+0000","last_became_active":"2023-10-09T06:46:48.520824+0000","last_became_peered":"2023-10-09T06:46:48.520824+0000","last_unstale":"2023-10-09T06:47:48.534984+0000","last_undegraded":"2023-10-09T06:47:48.534984+0000","last_fullsized":"2023-10-09T06:47:48.534984+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":340,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.538574+0000","last_change":"2023-10-09T06:45:33.542948+0000","last_active":"2023-10-09T06:47:48.538574+0000","last_peered":"2023-10-09T06:47:48.538574+0000","last_clean":"2023-10-09T06:47:48.538574+0000","last_became_active":"2023-10-09T06:45:33.542464+0000","last_became_peered":"2023-10-09T06:45:33.542464+0000","last_unstale":"2023-10-09T06:47:48.538574+0000","last_undegraded":"2023-10-09T06:47:48.538574+0000","last_fullsized":"2023-10-09T06:47:48.538574+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":18,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.534899+0000","last_change":"2023-10-09T06:46:48.521737+0000","last_active":"2023-10-09T06:47:48.534899+0000","last_peered":"2023-10-09T06:47:48.534899+0000","last_clean":"2023-10-09T06:47:48.534899+0000","last_became_active":"2023-10-09T06:46:48.520820+0000","last_became_peered":"2023-10-09T06:46:48.520820+0000","last_unstale":"2023-10-09T06:47:48.534899+0000","last_undegraded":"2023-10-09T06:47:48.534899+0000","last_fullsized":"2023-10-09T06:47:48.534899+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":316,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.535042+0000","last_change":"2023-10-09T06:46:48.530572+0000","last_active":"2023-10-09T06:47:48.535042+0000","last_peered":"2023-10-09T06:47:48.535042+0000","last_clean":"2023-10-09T06:47:48.535042+0000","last_became_active":"2023-10-09T06:46:48.525656+0000","last_became_peered":"2023-10-09T06:46:48.525656+0000","last_unstale":"2023-10-09T06:47:48.535042+0000","last_undegraded":"2023-10-09T06:47:48.535042+0000","last_fullsized":"2023-10-09T06:47:48.535042+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":337,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T06:47:48.530705+0000","last_change":"2023-10-09T06:47:08.517897+0000","last_active":"2023-10-09T06:47:48.530705+0000","last_peered":"2023-10-09T06:47:48.530705+0000","last_clean":"2023-10-09T06:47:48.530705+0000","last_became_active":"2023-10-09T06:47:08.517515+0000","last_became_peered":"2023-10-09T06:47:08.517515+0000","last_unstale":"2023-10-09T06:47:48.530705+0000","last_undegraded":"2023-10-09T06:47:48.530705+0000","last_fullsized":"2023-10-09T06:47:48.530705+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":94,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3047686168,"num_objects":46505,"num_object_clones":0,"num_object_copies":139515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7625,"num_objects_unfound":0,"num_objects_dirty":46505,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149263,"num_write_kb":6264513,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11441,"num_bytes_recovered":749797376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9691738112,"data_stored":9691725896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73327,"ondisk_log_size":73327,"up":47,"acting":48,"num_store_stats":5},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":79,"num_bytes_recovered":31380,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":93,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5929,"num_read_kb":12159,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":262,"num_bytes_recovered":4194399,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430803,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2842692,"kb_used_data":1936508,"kb_used_omap":0,"kb_used_meta":906112,"kb_avail":90902460,"statfs":{"total":95995035648,"available":93084119040,"internally_reserved":0,"allocated":1982984192,"data_stored":1982365003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927858688},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.173,"15min":1.1579999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.1310000000000002,"5min":6.1310000000000002,"15min":9.4540000000000006},"last":1.946},{"interface":"front","average":{"1min":1.9239999999999999,"5min":1.345,"15min":1.1200000000000001},"min":{"1min":0.35199999999999998,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":12.583,"5min":12.583,"15min":12.583},"last":2.1240000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:47:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.226,"15min":1.9139999999999999},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.27800000000000002},"max":{"1min":6.2359999999999998,"5min":6.2359999999999998,"15min":148.88499999999999},"last":1.98},{"interface":"front","average":{"1min":1.742,"5min":1.2490000000000001,"15min":1.8100000000000001},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.27200000000000002},"max":{"1min":6.2910000000000004,"5min":6.2910000000000004,"15min":148.059},"last":2.0329999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:48:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1739999999999999,"5min":1.72,"15min":1.619},"min":{"1min":0.41899999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.9790000000000001,"5min":9.0679999999999996,"15min":15.957000000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.3240000000000001,"15min":1.2629999999999999},"min":{"1min":0.36299999999999999,"5min":0.34699999999999998,"15min":0.33000000000000002},"max":{"1min":6.2380000000000004,"5min":6.2380000000000004,"15min":6.2380000000000004},"last":2.0030000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:47:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":2.2930000000000001,"15min":2.1890000000000001},"min":{"1min":0.45000000000000001,"5min":0.38,"15min":0.30499999999999999},"max":{"1min":9.0449999999999999,"5min":23.074000000000002,"15min":32.715000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.651,"5min":1.7170000000000001,"15min":1.8080000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.2619999999999996,"5min":23.163,"15min":32.728999999999999},"last":2.1890000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":1.337,"15min":1.163},"min":{"1min":0.39200000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.711,"5min":12.711,"15min":12.711},"last":2.089},{"interface":"front","average":{"1min":3.4159999999999999,"5min":1.8759999999999999,"15min":1.6040000000000001},"min":{"1min":0.38400000000000001,"5min":0.34899999999999998,"15min":0.28699999999999998},"max":{"1min":10.465,"5min":10.795999999999999,"15min":16.064},"last":2.863}]},{"osd":5,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.2010000000000001,"15min":1.1799999999999999},"min":{"1min":0.51500000000000001,"5min":0.38300000000000001,"15min":0.34799999999999998},"max":{"1min":6.2629999999999999,"5min":6.2629999999999999,"15min":10.680999999999999},"last":2.0590000000000002},{"interface":"front","average":{"1min":1.659,"5min":1.1279999999999999,"15min":1.256},"min":{"1min":0.47299999999999998,"5min":0.42899999999999999,"15min":0.33900000000000002},"max":{"1min":12.675000000000001,"5min":12.675000000000001,"15min":12.675000000000001},"last":2.177}]},{"osd":6,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.5189999999999999,"15min":1.3180000000000001},"min":{"1min":0.378,"5min":0.32200000000000001,"15min":0.314},"max":{"1min":12.640000000000001,"5min":12.640000000000001,"15min":12.640000000000001},"last":2.1509999999999998},{"interface":"front","average":{"1min":3.476,"5min":1.97,"15min":1.667},"min":{"1min":0.42999999999999999,"5min":0.40500000000000003,"15min":0.378},"max":{"1min":11.911,"5min":11.911,"15min":16.024000000000001},"last":1.49}]}]},{"osd":6,"up_from":42,"seq":180388627035,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2760568,"kb_used_data":1868144,"kb_used_omap":0,"kb_used_meta":892416,"kb_avail":90984584,"statfs":{"total":95995035648,"available":93168214016,"internally_reserved":0,"allocated":1912979456,"data_stored":1912371837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913833984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":2.3210000000000002,"15min":1.5800000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.9749999999999996,"5min":103.94199999999999,"15min":103.94199999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.5680000000000001,"5min":2.3999999999999999,"15min":1.6850000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.26800000000000002},"max":{"1min":5.7830000000000004,"5min":102.621,"15min":102.621},"last":3.1480000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:47:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2390000000000001,"5min":1.3300000000000001,"15min":2.3100000000000001},"min":{"1min":0.38500000000000001,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":3.2050000000000001,"5min":6.3639999999999999,"15min":316.88099999999997},"last":4.0220000000000002},{"interface":"front","average":{"1min":1.78,"5min":1.2969999999999999,"15min":2.2400000000000002},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":6.0369999999999999,"5min":6.0369999999999999,"15min":316.84399999999999},"last":0.68000000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:47:15 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":4.516,"15min":2.5579999999999998},"min":{"1min":0.35899999999999999,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":4.1900000000000004,"5min":271.976,"15min":271.976},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.506,"5min":4.4290000000000003,"15min":2.6030000000000002},"min":{"1min":0.40500000000000003,"5min":0.316,"15min":0.30599999999999999},"max":{"1min":5.8789999999999996,"5min":271.863,"15min":271.863},"last":3.1920000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.4059999999999999,"15min":1.7},"min":{"1min":0.40300000000000002,"5min":0.39200000000000002,"15min":0.32700000000000001},"max":{"1min":6.0570000000000004,"5min":8.2520000000000007,"15min":27.491},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.7090000000000001,"15min":1.8320000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.7720000000000002,"5min":12.238,"15min":26.815000000000001},"last":3.996}]},{"osd":4,"last update":"Mon Oct 9 06:47:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":1.786,"15min":1.7629999999999999},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.27400000000000002},"max":{"1min":6.7569999999999997,"5min":8,"15min":80.231999999999999},"last":3.0910000000000002},{"interface":"front","average":{"1min":3.3260000000000001,"5min":1.7849999999999999,"15min":1.8069999999999999},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":20.196000000000002,"5min":20.196000000000002,"15min":81.085999999999999},"last":4.048}]},{"osd":5,"last update":"Mon Oct 9 06:47:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.2210000000000001,"15min":1.236},"min":{"1min":0.36399999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.8490000000000002,"5min":5.8540000000000001,"15min":10.601000000000001},"last":3.2229999999999999},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.2270000000000001,"15min":1.536},"min":{"1min":0.311,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":3.2330000000000001,"5min":5.7400000000000002,"15min":11.526},"last":3.9540000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:47:21 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.6819999999999999,"15min":1.431},"min":{"1min":0.53200000000000003,"5min":0.44900000000000001,"15min":0.35299999999999998},"max":{"1min":8.5719999999999992,"5min":8.5719999999999992,"15min":10.673},"last":3.7290000000000001},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.3360000000000001,"15min":1.2470000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.33100000000000002},"max":{"1min":6.0140000000000002,"5min":6.0140000000000002,"15min":9.0459999999999994},"last":0.51900000000000002}]}]},{"osd":1,"up_from":0,"seq":399431958944,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122470,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3652784,"kb_used_data":2753256,"kb_used_omap":8,"kb_used_meta":899511,"kb_avail":90092368,"statfs":{"total":95995035648,"available":92254584832,"internally_reserved":0,"allocated":2819334144,"data_stored":2818209323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":921099520},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:47:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.0529999999999999,"15min":1.621},"min":{"1min":0.26000000000000001,"5min":0.20000000000000001,"15min":0.187},"max":{"1min":1.603,"5min":9.0670000000000002,"15min":25.084},"last":5.9050000000000002},{"interface":"front","average":{"1min":0.628,"5min":1.056,"15min":1.593},"min":{"1min":0.39800000000000002,"5min":0.27800000000000002,"15min":0.20300000000000001},"max":{"1min":1.5449999999999999,"5min":7.6680000000000001,"15min":25.036999999999999},"last":1.4039999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:48:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.93000000000000005,"15min":1.3779999999999999},"min":{"1min":0.40400000000000003,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.3460000000000001,"5min":9.1539999999999999,"15min":12.154},"last":6.6680000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.84399999999999997,"15min":1.339},"min":{"1min":0.33200000000000002,"5min":0.25700000000000001,"15min":0.215},"max":{"1min":1.6459999999999999,"5min":6.1159999999999997,"15min":15.351000000000001},"last":1.4790000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:47:44 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.147,"15min":2.3650000000000002},"min":{"1min":0.32800000000000001,"5min":0.23300000000000001,"15min":0.216},"max":{"1min":0.95399999999999996,"5min":11.564,"15min":86.709999999999994},"last":2.851},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.1080000000000001,"15min":2.355},"min":{"1min":0.39800000000000002,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":1.778,"5min":11.537000000000001,"15min":87.109999999999999},"last":1.5369999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.569,"15min":1.4550000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.309},"max":{"1min":9.5459999999999994,"5min":9.5459999999999994,"15min":9.5459999999999994},"last":4.806},{"interface":"front","average":{"1min":1.851,"5min":1.395,"15min":1.474},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.31},"max":{"1min":11.948,"5min":11.948,"15min":11.948},"last":6.7729999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.2030000000000001,"15min":1.5209999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.27800000000000002},"max":{"1min":11.894,"5min":11.894,"15min":11.894},"last":6.8360000000000003},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.111,"15min":1.4330000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":1.8759999999999999,"5min":7.7430000000000003,"15min":10.419},"last":4.6520000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.4359999999999999,"15min":1.5529999999999999},"min":{"1min":0.53600000000000003,"5min":0.432,"15min":0.33900000000000002},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":10.377000000000001},"last":4.6900000000000004},{"interface":"front","average":{"1min":1.464,"5min":1.486,"15min":1.4450000000000001},"min":{"1min":0.45600000000000002,"5min":0.42999999999999999,"15min":0.379},"max":{"1min":6.1159999999999997,"5min":6.5869999999999997,"15min":9.4510000000000005},"last":4.7679999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:47:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.4690000000000001,"15min":1.6930000000000001},"min":{"1min":0.51400000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":5.8380000000000001,"5min":7.7039999999999997,"15min":13.259},"last":4.7160000000000002},{"interface":"front","average":{"1min":1.702,"5min":1.5660000000000001,"15min":1.641},"min":{"1min":0.42399999999999999,"5min":0.40799999999999997,"15min":0.35299999999999998},"max":{"1min":5.798,"5min":6.6120000000000001,"15min":13.428000000000001},"last":4.8319999999999999}]}]},{"osd":2,"up_from":0,"seq":420906795418,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":442381631892,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019501,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3890240,"kb_used_data":2994744,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":89854912,"statfs":{"total":95995035648,"available":92011429888,"internally_reserved":0,"allocated":3066617856,"data_stored":3066002224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:47:17 2023","interfaces":[{"interface":"back","average":{"1min":4.3959999999999999,"5min":3.7149999999999999,"15min":2.6150000000000002},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.249},"max":{"1min":64.230999999999995,"5min":89,"15min":89},"last":4.5339999999999998},{"interface":"front","average":{"1min":5.1760000000000002,"5min":3.9369999999999998,"15min":2.7290000000000001},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.30399999999999999},"max":{"1min":64.463999999999999,"5min":89.430999999999997,"15min":89.430999999999997},"last":0.46700000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.165,"15min":1.353},"min":{"1min":0.34799999999999998,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":5.157,"5min":7.0659999999999998,"15min":17.033000000000001},"last":4.5910000000000002},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.94199999999999995,"15min":1.3799999999999999},"min":{"1min":0.376,"5min":0.33500000000000002,"15min":0.316},"max":{"1min":4.4359999999999999,"5min":4.4359999999999999,"15min":17.015999999999998},"last":3.8159999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:47:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":5.4379999999999997,"15min":3.3050000000000002},"min":{"1min":0.45000000000000001,"5min":0.28799999999999998,"15min":0.28599999999999998},"max":{"1min":10.739000000000001,"5min":231.393,"15min":231.393},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":4.8040000000000003,"15min":3.1190000000000002},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.4800000000000004,"5min":231.17099999999999,"15min":231.17099999999999},"last":3.9500000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:47:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":5.21,"15min":4.2869999999999999},"min":{"1min":0.40100000000000002,"5min":0.34699999999999998,"15min":0.34100000000000003},"max":{"1min":4.5490000000000004,"5min":234.62200000000001,"15min":234.62200000000001},"last":3.8969999999999998},{"interface":"front","average":{"1min":1.613,"5min":5.4420000000000002,"15min":4.3869999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.33900000000000002},"max":{"1min":5.3049999999999997,"5min":234.673,"15min":234.673},"last":4.5629999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.208,"15min":1.2529999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":5.2119999999999997,"5min":7.0330000000000004,"15min":22.068000000000001},"last":4.4980000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":1.4470000000000001,"15min":1.4019999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3719999999999999,"5min":10.707000000000001,"15min":13.095000000000001},"last":0.49199999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:47:32 2023","interfaces":[{"interface":"back","average":{"1min":2.9159999999999999,"5min":2.0049999999999999,"15min":1.544},"min":{"1min":0.58299999999999996,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":13.163,"5min":13.163,"15min":13.163},"last":0.437},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.375,"15min":1.294},"min":{"1min":0.625,"5min":0.38600000000000001,"15min":0.32300000000000001},"max":{"1min":5.1950000000000003,"5min":8.3900000000000006,"15min":11.782},"last":4.4390000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:47:32 2023","interfaces":[{"interface":"back","average":{"1min":3.1469999999999998,"5min":2.3029999999999999,"15min":1.7130000000000001},"min":{"1min":0.58299999999999996,"5min":0.504,"15min":0.35899999999999999},"max":{"1min":7.7809999999999997,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":3.8849999999999998},{"interface":"front","average":{"1min":1.6899999999999999,"5min":1.123,"15min":1.286},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.317},"max":{"1min":8.3239999999999998,"5min":8.3239999999999998,"15min":9.1370000000000005},"last":3.984}]}]},{"osd":5,"up_from":0,"seq":154618823268,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2779451392,"data_stored":2779447320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3037794304,"data_stored":3037790232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1901137920,"data_stored":1901133848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1973354496,"data_stored":1973354496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:48:18.324 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-09T06:48:18.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:18.397 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:18.500 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:18.525 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:18.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:18.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:18.678 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:18.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:18.784 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:18.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: audit 2023-10-09T06:48:16.628076+0000 mgr.y (mgr.25723) 1316 : audit 0 from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:48:18.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: cluster 2023-10-09T06:48:17.452569+0000 mgr.y (mgr.25723) 1317 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 659 op/s; 7629/139599 objects misplaced (5.465%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:48:18.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: cluster 2023-10-09T06:48:17.514637+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:18.795 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: audit 2023-10-09T06:48:17.525477+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: cluster 2023-10-09T06:48:17.525527+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e311: 8 total, 8 up, 4 in 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[80002]: cluster 2023-10-09T06:48:17.527575+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: audit 2023-10-09T06:48:16.628076+0000 mgr.y (mgr.25723) 1316 : audit 0 from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: cluster 2023-10-09T06:48:17.452569+0000 mgr.y (mgr.25723) 1317 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 659 op/s; 7629/139599 objects misplaced (5.465%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: cluster 2023-10-09T06:48:17.514637+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:18.796 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: audit 2023-10-09T06:48:17.525477+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:18.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: cluster 2023-10-09T06:48:17.525527+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e311: 8 total, 8 up, 4 in 2023-10-09T06:48:18.797 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:18 smithi125 bash[77299]: cluster 2023-10-09T06:48:17.527575+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T06:48:18.810 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:18.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:18.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: audit 2023-10-09T06:48:16.628076+0000 mgr.y (mgr.25723) 1316 : audit 0 from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:48:18.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: cluster 2023-10-09T06:48:17.452569+0000 mgr.y (mgr.25723) 1317 : cluster 0 pgmap v1428: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 659 op/s; 7629/139599 objects misplaced (5.465%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:48:18.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: cluster 2023-10-09T06:48:17.514637+0000 mon.a (mon.0) 1617 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:18.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: audit 2023-10-09T06:48:17.525477+0000 mon.a (mon.0) 1618 : audit 1 from='client.? 172.21.15.125:0/1211129408' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:18.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: cluster 2023-10-09T06:48:17.525527+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e311: 8 total, 8 up, 4 in 2023-10-09T06:48:18.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:18 smithi155 bash[59495]: cluster 2023-10-09T06:48:17.527575+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e311 with expected crc 2023-10-09T06:48:18.913 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:18.939 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:18.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:18.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:19.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:19.069 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:19.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:19.173 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:19.300 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:19.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:19.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:19.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:19.569 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:19.578 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:19 smithi155 bash[59495]: audit 2023-10-09T06:48:18.610417+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:19.586 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:19.587 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:19.600 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:19.708 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.738 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:19.843 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.857 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:19 smithi125 bash[77299]: audit 2023-10-09T06:48:18.610417+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:19.857 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:19 smithi125 bash[80002]: audit 2023-10-09T06:48:18.610417+0000 mon.a (mon.0) 1620 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:19.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:19.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:19.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:19.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:20.007 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:20.113 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:20.139 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:20.245 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:20.282 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:20.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:20.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:20.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:20.512 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:20.542 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:20.646 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:20.654 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:20 smithi125 bash[77299]: cluster 2023-10-09T06:48:19.453078+0000 mgr.y (mgr.25723) 1318 : cluster 0 pgmap v1430: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 7593/138837 objects misplaced (5.469%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:20.654 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:20 smithi125 bash[80002]: cluster 2023-10-09T06:48:19.453078+0000 mgr.y (mgr.25723) 1318 : cluster 0 pgmap v1430: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 7593/138837 objects misplaced (5.469%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:20.668 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:20.771 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:20.791 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:20 smithi155 bash[59495]: cluster 2023-10-09T06:48:19.453078+0000 mgr.y (mgr.25723) 1318 : cluster 0 pgmap v1430: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 7593/138837 objects misplaced (5.469%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:20.894 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:20.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:20.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:21.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:21.044 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:21.147 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.167 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:48:21.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:21.302 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.333 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:21.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:21.467 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:21.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:21.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.608 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:21.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:21.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:21] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.33.4" 2023-10-09T06:48:21.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:21.739 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:21.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:21.875 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:21.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:21.982 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:22.001 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:22.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:22.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:22.237 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:22.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:22.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:22.364 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:22.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:22.492 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:22.528 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:22.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:22 smithi125 bash[77299]: cluster 2023-10-09T06:48:21.453808+0000 mgr.y (mgr.25723) 1319 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 790 op/s; 6650/127278 objects misplaced (5.225%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:22 smithi125 bash[80002]: cluster 2023-10-09T06:48:21.453808+0000 mgr.y (mgr.25723) 1319 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 790 op/s; 6650/127278 objects misplaced (5.225%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:22.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:22.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:22.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:22.784 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:22.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:22.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:22 smithi155 bash[59495]: cluster 2023-10-09T06:48:21.453808+0000 mgr.y (mgr.25723) 1319 : cluster 0 pgmap v1431: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 790 op/s; 6650/127278 objects misplaced (5.225%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:22.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:22.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:22.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:23.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:23.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:23.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:23.272 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.315 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:23.420 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:23.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:23.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:23.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:23.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:23.697 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:23.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:23.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.873 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:23.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:23.979 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:24.018 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:24.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:24.149 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:24.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.279 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:24.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:24.406 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:24.408 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:24.516 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.541 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:24.646 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:24 smithi155 bash[59495]: cluster 2023-10-09T06:48:23.454368+0000 mgr.y (mgr.25723) 1320 : cluster 0 pgmap v1432: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 6598/126543 objects misplaced (5.214%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:24.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:24.675 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:24.780 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.788 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:24 smithi125 bash[77299]: cluster 2023-10-09T06:48:23.454368+0000 mgr.y (mgr.25723) 1320 : cluster 0 pgmap v1432: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 6598/126543 objects misplaced (5.214%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:24.789 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:24 smithi125 bash[80002]: cluster 2023-10-09T06:48:23.454368+0000 mgr.y (mgr.25723) 1320 : cluster 0 pgmap v1432: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 792 op/s; 6598/126543 objects misplaced (5.214%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:48:24.804 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:24.907 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:24.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:24.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:25.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:25.166 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.192 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:25.295 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:25.341 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:25.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.470 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:25.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:25.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:25.713 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:25.741 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:25.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:25.846 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:25.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:25.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:25.973 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:25.998 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:26.104 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:26.126 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:26.228 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:26.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:26.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:26.389 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:26.493 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:26.521 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:26.535 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:26 smithi155 bash[59495]: cluster 2023-10-09T06:48:25.455349+0000 mgr.y (mgr.25723) 1321 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:26.629 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:26.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:26 smithi125 bash[77299]: cluster 2023-10-09T06:48:25.455349+0000 mgr.y (mgr.25723) 1321 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:26.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:26 smithi125 bash[80002]: cluster 2023-10-09T06:48:25.455349+0000 mgr.y (mgr.25723) 1321 : cluster 0 pgmap v1433: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:26.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:26.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:26.786 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:26.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:26.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:26.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:26.986 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:27.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:48:27.025 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:48:27.025 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:48:27.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:27.052 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:27.057 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:48:27.090 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:48:27.116 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:48:27.143 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:48:27.155 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:27.172 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:48:27.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:27.206 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:48:27.231 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:48:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.233 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.234 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:48:27.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:27.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:27.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:27.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:27.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:27 smithi155 bash[59495]: audit 2023-10-09T06:48:26.979561+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.125:0/695170216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:27.545 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:27.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:27.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:27.671 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:27.700 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:27.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:27.803 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:27.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:27.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:27 smithi125 bash[77299]: audit 2023-10-09T06:48:26.979561+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.125:0/695170216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:27 smithi125 bash[80002]: audit 2023-10-09T06:48:26.979561+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.125:0/695170216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:27.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:27.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:27.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:27.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:28.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:28.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:28.213 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:28.235 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:28.339 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:28.364 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:28.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:28.468 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:28.510 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:28.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:28 smithi125 bash[77299]: cluster 2023-10-09T06:48:27.455843+0000 mgr.y (mgr.25723) 1322 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:28.623 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:28 smithi125 bash[80002]: cluster 2023-10-09T06:48:27.455843+0000 mgr.y (mgr.25723) 1322 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:28.639 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:28.742 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:28.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:28.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:28 smithi155 bash[59495]: cluster 2023-10-09T06:48:27.455843+0000 mgr.y (mgr.25723) 1322 : cluster 0 pgmap v1434: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 836 op/s; 5684/114504 objects misplaced (4.964%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T06:48:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:28.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:28.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:29.039 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:29.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:29.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:29.181 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:29.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:29.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:29.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:29.447 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:29.552 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:29.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:29.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:29.729 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:29.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:29.833 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:29.858 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:29.963 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:29.993 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:29.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:29.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:30.098 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:30.134 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:30.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:30.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:30.250 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:30 smithi155 bash[59495]: cluster 2023-10-09T06:48:29.456377+0000 mgr.y (mgr.25723) 1323 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 5628/113664 objects misplaced (4.951%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:48:30.277 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:30.384 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:30.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:30.437 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:30 smithi125 bash[77299]: cluster 2023-10-09T06:48:29.456377+0000 mgr.y (mgr.25723) 1323 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 5628/113664 objects misplaced (4.951%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:48:30.437 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:30 smithi125 bash[80002]: cluster 2023-10-09T06:48:29.456377+0000 mgr.y (mgr.25723) 1323 : cluster 0 pgmap v1435: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 5628/113664 objects misplaced (4.951%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:48:30.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:48:30.526 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:30.588 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:30.692 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:30.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:30.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:30.819 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:30.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.862 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:30.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:30.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:30.984 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:30.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:31.088 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:31.113 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:31.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:31.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:31.349 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:31.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:31 smithi125 bash[77299]: audit 2023-10-09T06:48:30.438820+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:48:31.360 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:31] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.33.4" 2023-10-09T06:48:31.360 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:31 smithi125 bash[80002]: audit 2023-10-09T06:48:30.438820+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:48:31.377 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:31.480 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:31.513 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:31.616 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:31.626 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:31 smithi155 bash[59495]: audit 2023-10-09T06:48:30.438820+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:48:31.646 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:31.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:31.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:31.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:31.883 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:31.912 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:31.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:32.016 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:32.043 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:32.146 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:32.175 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:32.278 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:32.305 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:32.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:32.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:32 smithi155 bash[59495]: cluster 2023-10-09T06:48:31.152748+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:32 smithi155 bash[59495]: cluster 2023-10-09T06:48:31.152849+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:32 smithi155 bash[59495]: cluster 2023-10-09T06:48:31.153516+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e312: 8 total, 8 up, 4 in 2023-10-09T06:48:32.418 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:32 smithi155 bash[59495]: cluster 2023-10-09T06:48:31.155458+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.419 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:32 smithi155 bash[59495]: cluster 2023-10-09T06:48:31.456936+0000 mgr.y (mgr.25723) 1324 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 4721/102588 objects misplaced (4.602%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:32.437 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:32.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:32.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:48:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[77299]: cluster 2023-10-09T06:48:31.152748+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[77299]: cluster 2023-10-09T06:48:31.152849+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[77299]: cluster 2023-10-09T06:48:31.153516+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e312: 8 total, 8 up, 4 in 2023-10-09T06:48:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[77299]: cluster 2023-10-09T06:48:31.155458+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.501 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[77299]: cluster 2023-10-09T06:48:31.456936+0000 mgr.y (mgr.25723) 1324 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 4721/102588 objects misplaced (4.602%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:32.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[80002]: cluster 2023-10-09T06:48:31.152748+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[80002]: cluster 2023-10-09T06:48:31.152849+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[80002]: cluster 2023-10-09T06:48:31.153516+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e312: 8 total, 8 up, 4 in 2023-10-09T06:48:32.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[80002]: cluster 2023-10-09T06:48:31.155458+0000 osd.4 (osd.4) 267 : cluster 3 failed to encode map e312 with expected crc 2023-10-09T06:48:32.502 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:32 smithi125 bash[80002]: cluster 2023-10-09T06:48:31.456936+0000 mgr.y (mgr.25723) 1324 : cluster 0 pgmap v1437: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 4721/102588 objects misplaced (4.602%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:48:32.541 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:32.569 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:32.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:32.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:32.817 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:32.842 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:32.947 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:32.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:32.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:33.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:33.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:33.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:33.244 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:33.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:33.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:33 smithi155 bash[59495]: cluster 2023-10-09T06:48:32.161375+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e313: 8 total, 8 up, 4 in 2023-10-09T06:48:33.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:33 smithi155 bash[59495]: cluster 2023-10-09T06:48:32.163641+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.358 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:33 smithi155 bash[59495]: cluster 2023-10-09T06:48:32.487275+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.380 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:33.484 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:33.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[77299]: cluster 2023-10-09T06:48:32.161375+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e313: 8 total, 8 up, 4 in 2023-10-09T06:48:33.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[77299]: cluster 2023-10-09T06:48:32.163641+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[77299]: cluster 2023-10-09T06:48:32.487275+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[80002]: cluster 2023-10-09T06:48:32.161375+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e313: 8 total, 8 up, 4 in 2023-10-09T06:48:33.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[80002]: cluster 2023-10-09T06:48:32.163641+0000 osd.4 (osd.4) 268 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.496 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:33 smithi125 bash[80002]: cluster 2023-10-09T06:48:32.487275+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e313 with expected crc 2023-10-09T06:48:33.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:33.613 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:33.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:33.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:33.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:33.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:33.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:33.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:34.014 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:34.044 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:34.147 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.169 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:34.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:34 smithi125 bash[77299]: cluster 2023-10-09T06:48:33.457445+0000 mgr.y (mgr.25723) 1325 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 518 op/s; 4687/102063 objects misplaced (4.592%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:48:34.286 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:34 smithi125 bash[77299]: audit 2023-10-09T06:48:33.610277+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:34.286 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:34 smithi125 bash[80002]: cluster 2023-10-09T06:48:33.457445+0000 mgr.y (mgr.25723) 1325 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 518 op/s; 4687/102063 objects misplaced (4.592%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:48:34.287 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:34 smithi125 bash[80002]: audit 2023-10-09T06:48:33.610277+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:34.288 INFO:teuthology.orchestra.run.smithi125.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-09T06:48:34.301 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:34.406 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:34.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:34.532 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:34.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:34 smithi155 bash[59495]: cluster 2023-10-09T06:48:33.457445+0000 mgr.y (mgr.25723) 1325 : cluster 0 pgmap v1439: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 6 active+clean+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 518 op/s; 4687/102063 objects misplaced (4.592%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:48:34.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:34 smithi155 bash[59495]: audit 2023-10-09T06:48:33.610277+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:34.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:34.678 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:34.703 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:34.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:34.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:34.974 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:34.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:35.078 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.111 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:35.217 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:35.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:35.341 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.350 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:48:35.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:35 smithi155 bash[59495]: audit 2023-10-09T06:48:34.282064+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.125:0/1011632039' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:35 smithi155 bash[59495]: audit 2023-10-09T06:48:34.282482+0000 mon.a (mon.0) 1625 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.375 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:35.469 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:35 smithi125 bash[77299]: audit 2023-10-09T06:48:34.282064+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.125:0/1011632039' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:35 smithi125 bash[77299]: audit 2023-10-09T06:48:34.282482+0000 mon.a (mon.0) 1625 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:35 smithi125 bash[80002]: audit 2023-10-09T06:48:34.282064+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.125:0/1011632039' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:35 smithi125 bash[80002]: audit 2023-10-09T06:48:34.282482+0000 mon.a (mon.0) 1625 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-09T06:48:35.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:35.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:35.646 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:35.685 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:35.791 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:35.814 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:35.919 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:35.950 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:35.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:35.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:36.053 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:36.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:36.188 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:36.320 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:36.336 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:36 smithi155 bash[59495]: cluster 2023-10-09T06:48:35.458120+0000 mgr.y (mgr.25723) 1326 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 861 op/s; 3586/93825 objects misplaced (3.822%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:48:36.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:36.459 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:36 smithi125 bash[77299]: cluster 2023-10-09T06:48:35.458120+0000 mgr.y (mgr.25723) 1326 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 861 op/s; 3586/93825 objects misplaced (3.822%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:48:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-09T06:48:36.461 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:36 smithi125 bash[80002]: cluster 2023-10-09T06:48:35.458120+0000 mgr.y (mgr.25723) 1326 : cluster 0 pgmap v1440: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 861 op/s; 3586/93825 objects misplaced (3.822%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:48:36.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:36.491 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:36.596 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:36.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:36.735 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.765 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:36.869 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:36.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:36.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:36.929 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:37.003 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:37.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:37.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:37.175 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:37.200 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:48:37.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:37.304 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:37.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:37 smithi155 bash[59495]: audit 2023-10-09T06:48:37.132625+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.125:0/1180130756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:37 smithi155 bash[59495]: audit 2023-10-09T06:48:37.133043+0000 mon.a (mon.0) 1626 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:37.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:37.534 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:37.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:37 smithi125 bash[77299]: audit 2023-10-09T06:48:37.132625+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.125:0/1180130756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:37 smithi125 bash[77299]: audit 2023-10-09T06:48:37.133043+0000 mon.a (mon.0) 1626 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.545 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:37 smithi125 bash[80002]: audit 2023-10-09T06:48:37.132625+0000 mon.c (mon.2) 130 : audit 1 from='client.? 172.21.15.125:0/1180130756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.545 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:37 smithi125 bash[80002]: audit 2023-10-09T06:48:37.133043+0000 mon.a (mon.0) 1626 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:48:37.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:37.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:37.698 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:37.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:37.837 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:37.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:37.975 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:37.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:38.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.104 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:38.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:38.337 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:38.359 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:38.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: cluster 2023-10-09T06:48:37.178728+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:38.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: cluster 2023-10-09T06:48:37.191999+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T06:48:38.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: audit 2023-10-09T06:48:37.193604+0000 mon.a (mon.0) 1628 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: cluster 2023-10-09T06:48:37.193678+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e314: 8 total, 8 up, 4 in 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: cluster 2023-10-09T06:48:37.458716+0000 mgr.y (mgr.25723) 1327 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 3586/93825 objects misplaced (3.822%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: audit 2023-10-09T06:48:37.873092+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: audit 2023-10-09T06:48:37.880576+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: audit 2023-10-09T06:48:38.025379+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:38 smithi155 bash[59495]: audit 2023-10-09T06:48:38.032089+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: cluster 2023-10-09T06:48:37.178728+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:38.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: cluster 2023-10-09T06:48:37.191999+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T06:48:38.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: audit 2023-10-09T06:48:37.193604+0000 mon.a (mon.0) 1628 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:48:38.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: cluster 2023-10-09T06:48:37.193678+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e314: 8 total, 8 up, 4 in 2023-10-09T06:48:38.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: cluster 2023-10-09T06:48:37.458716+0000 mgr.y (mgr.25723) 1327 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 3586/93825 objects misplaced (3.822%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: audit 2023-10-09T06:48:37.873092+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: audit 2023-10-09T06:48:37.880576+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: audit 2023-10-09T06:48:38.025379+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[77299]: audit 2023-10-09T06:48:38.032089+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: cluster 2023-10-09T06:48:37.178728+0000 mon.a (mon.0) 1627 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: cluster 2023-10-09T06:48:37.191999+0000 osd.4 (osd.4) 269 : cluster 3 failed to encode map e314 with expected crc 2023-10-09T06:48:38.423 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: audit 2023-10-09T06:48:37.193604+0000 mon.a (mon.0) 1628 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: cluster 2023-10-09T06:48:37.193678+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e314: 8 total, 8 up, 4 in 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: cluster 2023-10-09T06:48:37.458716+0000 mgr.y (mgr.25723) 1327 : cluster 0 pgmap v1442: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 3586/93825 objects misplaced (3.822%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: audit 2023-10-09T06:48:37.873092+0000 mon.a (mon.0) 1630 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: audit 2023-10-09T06:48:37.880576+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: audit 2023-10-09T06:48:38.025379+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.424 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:38 smithi125 bash[80002]: audit 2023-10-09T06:48:38.032089+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:38.468 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.493 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:38.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:38.599 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:38.629 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:38.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:38.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:38.764 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:38.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:38.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:38.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:38.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:38.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:39.126 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.150 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:39.253 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:39.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:39.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:39.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:39 smithi155 bash[59495]: audit 2023-10-09T06:48:38.511844+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:48:39.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:39 smithi155 bash[59495]: audit 2023-10-09T06:48:38.512930+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:48:39.395 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:39 smithi155 bash[59495]: audit 2023-10-09T06:48:38.519119+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:39.401 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:39.504 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.530 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:39.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:39.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[77299]: audit 2023-10-09T06:48:38.511844+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:48:39.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[77299]: audit 2023-10-09T06:48:38.512930+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:48:39.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[77299]: audit 2023-10-09T06:48:38.519119+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:39.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[80002]: audit 2023-10-09T06:48:38.511844+0000 mon.a (mon.0) 1634 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:48:39.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[80002]: audit 2023-10-09T06:48:38.512930+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:48:39.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:39 smithi125 bash[80002]: audit 2023-10-09T06:48:38.519119+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:39.656 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:39.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:39.789 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:39.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:39.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:40.029 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:40.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:40.155 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.185 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:40.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:40.286 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:40.332 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:40.438 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:40.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:40 smithi155 bash[59495]: cluster 2023-10-09T06:48:39.459282+0000 mgr.y (mgr.25723) 1328 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 396 op/s; 3545/93078 objects misplaced (3.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:48:40.469 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:40.574 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:40.583 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:40 smithi125 bash[77299]: cluster 2023-10-09T06:48:39.459282+0000 mgr.y (mgr.25723) 1328 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 396 op/s; 3545/93078 objects misplaced (3.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:48:40.583 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:40 smithi125 bash[80002]: cluster 2023-10-09T06:48:39.459282+0000 mgr.y (mgr.25723) 1328 : cluster 0 pgmap v1443: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.9 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 396 op/s; 3545/93078 objects misplaced (3.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:48:40.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:40.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:40.734 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:40.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:40.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:40.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:40.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:40.982 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:41.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:41.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:41.113 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:41.138 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:41.241 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:41.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:41.377 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:41.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:41.462 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 7, 6] out_osds: [5, 1, 2, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:48:41.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:48:41.471 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:41] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-10-09T06:48:41.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:41.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:41.636 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:41.669 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:41.773 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:41.807 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:41.847 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:48:41.912 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:41.942 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:41.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:42.049 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:42.070 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:42.175 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:42.208 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:42.316 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:42.353 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:42.456 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:42.482 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:42.586 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:42.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:42 smithi155 bash[59495]: cluster 2023-10-09T06:48:41.460380+0000 mgr.y (mgr.25723) 1329 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 2756/82317 objects misplaced (3.348%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:48:42.712 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:42.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:42.867 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:42 smithi125 bash[77299]: cluster 2023-10-09T06:48:41.460380+0000 mgr.y (mgr.25723) 1329 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 2756/82317 objects misplaced (3.348%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:48:42.867 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:42 smithi125 bash[80002]: cluster 2023-10-09T06:48:41.460380+0000 mgr.y (mgr.25723) 1329 : cluster 0 pgmap v1444: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 2756/82317 objects misplaced (3.348%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:48:42.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:42.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:42.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:43.006 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:43.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:43.083 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:48:43.110 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:48:43.135 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:48:43.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.157 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:48:43.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:43.201 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:48:43.230 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:48:43.261 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:48:43.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.292 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.295 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.296 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:48:43.302 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:43.404 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:43.428 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:43.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:43.534 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:43 smithi155 bash[59495]: audit 2023-10-09T06:48:43.051966+0000 mon.a (mon.0) 1637 : audit 0 from='client.? 172.21.15.125:0/3633392905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:43.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:43.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:43 smithi125 bash[77299]: audit 2023-10-09T06:48:43.051966+0000 mon.a (mon.0) 1637 : audit 0 from='client.? 172.21.15.125:0/3633392905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:43 smithi125 bash[80002]: audit 2023-10-09T06:48:43.051966+0000 mon.a (mon.0) 1637 : audit 0 from='client.? 172.21.15.125:0/3633392905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:48:43.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:43.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:43.797 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:43.822 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:43.925 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.953 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:43.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:44.056 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.078 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:44.180 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.215 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:44.321 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:44.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:44.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:44.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:44.583 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:44.600 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:44 smithi155 bash[59495]: cluster 2023-10-09T06:48:43.460859+0000 mgr.y (mgr.25723) 1330 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 684 op/s; 2706/81510 objects misplaced (3.320%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:48:44.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:44 smithi125 bash[77299]: cluster 2023-10-09T06:48:43.460859+0000 mgr.y (mgr.25723) 1330 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 684 op/s; 2706/81510 objects misplaced (3.320%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:48:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:44 smithi125 bash[80002]: cluster 2023-10-09T06:48:43.460859+0000 mgr.y (mgr.25723) 1330 : cluster 0 pgmap v1445: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.7 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 684 op/s; 2706/81510 objects misplaced (3.320%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:48:44.711 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.745 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:44.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:44.827 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:48:44.827 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:48:44.849 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:44.881 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:44.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:44.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.014 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:45.122 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:45.153 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:45.256 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:45.397 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:45.424 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:45.527 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:45.559 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:45.666 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:45.694 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:45.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:45.824 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:45.927 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.959 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:45.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:46.063 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:46.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:46.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:46.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:48:46.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:46.256 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:46.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:46.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:48:46.393 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:46.498 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:46.526 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:46.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:46.631 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:46 smithi125 bash[77299]: cluster 2023-10-09T06:48:45.461931+0000 mgr.y (mgr.25723) 1331 : cluster 0 pgmap v1446: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 1968/70449 objects misplaced (2.794%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:46.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:46 smithi125 bash[80002]: cluster 2023-10-09T06:48:45.461931+0000 mgr.y (mgr.25723) 1331 : cluster 0 pgmap v1446: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 1968/70449 objects misplaced (2.794%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:46.654 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:46.757 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:46.783 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:46.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:46.887 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:46.890 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:46.890 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:46 smithi155 bash[59495]: cluster 2023-10-09T06:48:45.461931+0000 mgr.y (mgr.25723) 1331 : cluster 0 pgmap v1446: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 1968/70449 objects misplaced (2.794%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:48:46.919 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:46.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:47.030 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:47.057 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:47.161 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:47.185 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:47.288 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:47.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:47.429 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:47.456 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:47.562 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.594 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:47.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:47.726 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:47.830 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:47 smithi155 bash[59495]: cluster 2023-10-09T06:48:47.462338+0000 mgr.y (mgr.25723) 1332 : cluster 0 pgmap v1447: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 1968/70449 objects misplaced (2.794%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:48:47.839 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:47 smithi155 bash[59495]: cluster 2023-10-09T06:48:47.548779+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e315: 8 total, 8 up, 4 in 2023-10-09T06:48:47.856 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:47.902 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:47 smithi125 bash[77299]: cluster 2023-10-09T06:48:47.462338+0000 mgr.y (mgr.25723) 1332 : cluster 0 pgmap v1447: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 1968/70449 objects misplaced (2.794%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:48:47.902 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:47 smithi125 bash[77299]: cluster 2023-10-09T06:48:47.548779+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e315: 8 total, 8 up, 4 in 2023-10-09T06:48:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:47 smithi125 bash[80002]: cluster 2023-10-09T06:48:47.462338+0000 mgr.y (mgr.25723) 1332 : cluster 0 pgmap v1447: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 1968/70449 objects misplaced (2.794%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:48:47.903 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:47 smithi125 bash[80002]: cluster 2023-10-09T06:48:47.548779+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e315: 8 total, 8 up, 4 in 2023-10-09T06:48:47.960 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.988 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:47.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:48.091 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:48.122 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:48.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:48.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:48.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:48.355 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:48.387 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:48.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:48.521 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:48.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:48.624 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:48.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:48 smithi155 bash[59495]: cluster 2023-10-09T06:48:47.553240+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.633 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:48 smithi155 bash[59495]: cluster 2023-10-09T06:48:47.555142+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.634 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:48 smithi155 bash[59495]: cluster 2023-10-09T06:48:48.552772+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e316: 8 total, 8 up, 4 in 2023-10-09T06:48:48.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:48.765 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:48.790 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:48.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[80002]: cluster 2023-10-09T06:48:47.553240+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[80002]: cluster 2023-10-09T06:48:47.555142+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[80002]: cluster 2023-10-09T06:48:48.552772+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e316: 8 total, 8 up, 4 in 2023-10-09T06:48:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[77299]: cluster 2023-10-09T06:48:47.553240+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[77299]: cluster 2023-10-09T06:48:47.555142+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e315 with expected crc 2023-10-09T06:48:48.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:48 smithi125 bash[77299]: cluster 2023-10-09T06:48:48.552772+0000 mon.a (mon.0) 1639 : cluster 0 osdmap e316: 8 total, 8 up, 4 in 2023-10-09T06:48:48.895 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:48.923 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:48.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:49.026 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.057 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:49.160 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:49.189 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:49.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.325 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:49.429 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:49.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:49.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:49.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:49.685 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.726 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[77299]: cluster 2023-10-09T06:48:48.554495+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[77299]: audit 2023-10-09T06:48:48.610819+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[77299]: cluster 2023-10-09T06:48:48.953127+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[77299]: cluster 2023-10-09T06:48:49.462906+0000 mgr.y (mgr.25723) 1333 : cluster 0 pgmap v1450: 89 pgs: 2 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 525 op/s; 1804/69696 objects misplaced (2.588%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[77299]: cluster 2023-10-09T06:48:49.551842+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e317: 8 total, 8 up, 4 in 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[80002]: cluster 2023-10-09T06:48:48.554495+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.753 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[80002]: audit 2023-10-09T06:48:48.610819+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:49.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[80002]: cluster 2023-10-09T06:48:48.953127+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[80002]: cluster 2023-10-09T06:48:49.462906+0000 mgr.y (mgr.25723) 1333 : cluster 0 pgmap v1450: 89 pgs: 2 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 525 op/s; 1804/69696 objects misplaced (2.588%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:48:49.754 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:49 smithi125 bash[80002]: cluster 2023-10-09T06:48:49.551842+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e317: 8 total, 8 up, 4 in 2023-10-09T06:48:49.831 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:49 smithi155 bash[59495]: cluster 2023-10-09T06:48:48.554495+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:49 smithi155 bash[59495]: audit 2023-10-09T06:48:48.610819+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:48:49.840 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:49 smithi155 bash[59495]: cluster 2023-10-09T06:48:48.953127+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e316 with expected crc 2023-10-09T06:48:49.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:49 smithi155 bash[59495]: cluster 2023-10-09T06:48:49.462906+0000 mgr.y (mgr.25723) 1333 : cluster 0 pgmap v1450: 89 pgs: 2 active+remapped+backfilling, 6 active+clean+remapped, 81 active+clean; 1.4 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 525 op/s; 1804/69696 objects misplaced (2.588%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:48:49.841 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:49 smithi155 bash[59495]: cluster 2023-10-09T06:48:49.551842+0000 mon.a (mon.0) 1641 : cluster 0 osdmap e317: 8 total, 8 up, 4 in 2023-10-09T06:48:49.857 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:49.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:49.965 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:49.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:49.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:49.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:50.096 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.130 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:50.233 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:50.361 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.388 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:50.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:50.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:50 smithi155 bash[59495]: cluster 2023-10-09T06:48:49.549296+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:50 smithi155 bash[59495]: cluster 2023-10-09T06:48:49.553586+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.654 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:50.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:50.782 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:50.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:50 smithi125 bash[77299]: cluster 2023-10-09T06:48:49.549296+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:50 smithi125 bash[77299]: cluster 2023-10-09T06:48:49.553586+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:50 smithi125 bash[80002]: cluster 2023-10-09T06:48:49.549296+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:50 smithi125 bash[80002]: cluster 2023-10-09T06:48:49.553586+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e317 with expected crc 2023-10-09T06:48:50.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:50.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:51.001 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:48:51.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:51.033 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:51.138 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:51.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:51.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:51.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:51.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:51.440 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:51.546 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:51.582 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:51.604 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:48:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:48:51] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-10-09T06:48:51.604 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:51 smithi125 bash[80002]: cluster 2023-10-09T06:48:51.463717+0000 mgr.y (mgr.25723) 1334 : cluster 0 pgmap v1452: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:51.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:51.699 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:51 smithi155 bash[59495]: cluster 2023-10-09T06:48:51.463717+0000 mgr.y (mgr.25723) 1334 : cluster 0 pgmap v1452: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:51.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:51.830 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:51.856 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:51.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:51 smithi125 bash[77299]: cluster 2023-10-09T06:48:51.463717+0000 mgr.y (mgr.25723) 1334 : cluster 0 pgmap v1452: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:51.959 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:51.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:51.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:51.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:52.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:52.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:52.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:52.240 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:48:52.247 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:52.352 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:52.376 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:52.481 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:52.508 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:52.611 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:52.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:52.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:52.742 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:52.776 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:52.881 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:52.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:52.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:53.014 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:53.048 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:53.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:53.151 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:53.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:53.288 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:53.310 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:53.414 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:53.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:53.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:53.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:53.567 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:53.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:53.671 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:53.696 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:53.801 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:53.826 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:53.931 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:53.962 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:53.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:54.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:54.095 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:54.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:54.201 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:54.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:54.348 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:54.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.476 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:54.523 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:54 smithi155 bash[59495]: cluster 2023-10-09T06:48:53.464100+0000 mgr.y (mgr.25723) 1335 : cluster 0 pgmap v1453: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:54.578 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:54.605 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:54.712 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:54.735 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:54.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:54.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:54 smithi125 bash[77299]: cluster 2023-10-09T06:48:53.464100+0000 mgr.y (mgr.25723) 1335 : cluster 0 pgmap v1453: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:54.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:54 smithi125 bash[80002]: cluster 2023-10-09T06:48:53.464100+0000 mgr.y (mgr.25723) 1335 : cluster 0 pgmap v1453: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 507 op/s; 1314/61308 objects misplaced (2.143%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:48:54.838 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:54.864 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:54.967 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:54.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:55.092 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.117 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:55.223 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:55.246 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:55.350 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:55.385 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:55.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.535 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:55.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.677 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:55.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:55.814 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:55.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:55.949 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:55.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:56.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:56.084 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:56.187 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.210 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:56.314 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:56.338 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:56.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:56.467 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:56.536 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:48:56.541 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:48:56.570 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.592 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:56.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:56 smithi155 bash[59495]: cluster 2023-10-09T06:48:55.464758+0000 mgr.y (mgr.25723) 1336 : cluster 0 pgmap v1454: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 897 op/s; 1073/49119 objects misplaced (2.184%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:48:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:56 smithi155 bash[59495]: audit 2023-10-09T06:48:55.830477+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.125:0/1566310933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:56 smithi155 bash[59495]: audit 2023-10-09T06:48:55.831050+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:56 smithi155 bash[59495]: audit 2023-10-09T06:48:56.221355+0000 mon.a (mon.0) 1643 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:56.695 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:56.725 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:56.828 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.850 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:56.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[77299]: cluster 2023-10-09T06:48:55.464758+0000 mgr.y (mgr.25723) 1336 : cluster 0 pgmap v1454: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 897 op/s; 1073/49119 objects misplaced (2.184%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:48:56.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[77299]: audit 2023-10-09T06:48:55.830477+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.125:0/1566310933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[77299]: audit 2023-10-09T06:48:55.831050+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.892 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[77299]: audit 2023-10-09T06:48:56.221355+0000 mon.a (mon.0) 1643 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:56.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[80002]: cluster 2023-10-09T06:48:55.464758+0000 mgr.y (mgr.25723) 1336 : cluster 0 pgmap v1454: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 897 op/s; 1073/49119 objects misplaced (2.184%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:48:56.892 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[80002]: audit 2023-10-09T06:48:55.830477+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.125:0/1566310933' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[80002]: audit 2023-10-09T06:48:55.831050+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:48:56.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:56 smithi125 bash[80002]: audit 2023-10-09T06:48:56.221355+0000 mon.a (mon.0) 1643 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:48:56.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:56.979 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:56.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:57.083 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:57.112 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:57.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:57.216 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.242 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:57.289 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:48:57.289 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:48:57.344 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:57.377 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:57.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:57.481 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.508 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:57.612 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:57.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.520630+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.520666+0000 mon.a (mon.0) 1645 : cluster 1 Cluster is now healthy 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: audit 2023-10-09T06:48:56.526959+0000 mon.a (mon.0) 1646 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: audit 2023-10-09T06:48:56.527089+0000 mon.a (mon.0) 1647 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.528962+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.530330+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.530672+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.530963+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e318: 8 total, 8 up, 4 in 2023-10-09T06:48:57.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:57 smithi155 bash[59495]: cluster 2023-10-09T06:48:56.827497+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.520630+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.520666+0000 mon.a (mon.0) 1645 : cluster 1 Cluster is now healthy 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: audit 2023-10-09T06:48:56.526959+0000 mon.a (mon.0) 1646 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: audit 2023-10-09T06:48:56.527089+0000 mon.a (mon.0) 1647 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.528962+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.530330+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.530672+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.745 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.530963+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e318: 8 total, 8 up, 4 in 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[77299]: cluster 2023-10-09T06:48:56.827497+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.520630+0000 mon.a (mon.0) 1644 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.520666+0000 mon.a (mon.0) 1645 : cluster 1 Cluster is now healthy 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: audit 2023-10-09T06:48:56.526959+0000 mon.a (mon.0) 1646 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: audit 2023-10-09T06:48:56.527089+0000 mon.a (mon.0) 1647 : audit 1 from='client.? 172.21.15.125:0/1245965726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.528962+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.530330+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.530672+0000 osd.4 (osd.4) 270 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.530963+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e318: 8 total, 8 up, 4 in 2023-10-09T06:48:57.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:57 smithi125 bash[80002]: cluster 2023-10-09T06:48:56.827497+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e318 with expected crc 2023-10-09T06:48:57.757 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:57.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:57.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.882 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:57.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:57.985 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:58.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:58.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:58.143 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:58.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:58.271 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:48:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:48:58.374 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:58.402 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:58.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:58.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:48:58.536 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.465354+0000 mgr.y (mgr.25723) 1337 : cluster 0 pgmap v1456: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 857 op/s; 1073/49119 objects misplaced (2.184%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.543493+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.545344+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e319: 8 total, 8 up, 4 in 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.547100+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.547158+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.548856+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[77299]: cluster 2023-10-09T06:48:57.550606+0000 osd.6 (osd.6) 283 : cluster 0 10.0 starting backfill to osd.0 from (0'0,0'0] MAX to 317'11237 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.465354+0000 mgr.y (mgr.25723) 1337 : cluster 0 pgmap v1456: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 857 op/s; 1073/49119 objects misplaced (2.184%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.543493+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.545344+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e319: 8 total, 8 up, 4 in 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.547100+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.547158+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.548856+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:58 smithi125 bash[80002]: cluster 2023-10-09T06:48:57.550606+0000 osd.6 (osd.6) 283 : cluster 0 10.0 starting backfill to osd.0 from (0'0,0'0] MAX to 317'11237 2023-10-09T06:48:58.640 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:58.666 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:58.771 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.465354+0000 mgr.y (mgr.25723) 1337 : cluster 0 pgmap v1456: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 857 op/s; 1073/49119 objects misplaced (2.184%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.543493+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.545344+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e319: 8 total, 8 up, 4 in 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.547100+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.547158+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.548856+0000 osd.4 (osd.4) 271 : cluster 3 failed to encode map e319 with expected crc 2023-10-09T06:48:58.787 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:58 smithi155 bash[59495]: cluster 2023-10-09T06:48:57.550606+0000 osd.6 (osd.6) 283 : cluster 0 10.0 starting backfill to osd.0 from (0'0,0'0] MAX to 317'11237 2023-10-09T06:48:58.798 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:48:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:48:58.904 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:58.931 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:48:58.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:58.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:48:59.034 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:59.054 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:59.158 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:59.181 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:48:59.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:48:59.284 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:59.309 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:59.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:59.452 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:48:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:48:59.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.546469+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.549278+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e320: 8 total, 8 up, 4 in 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.550807+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.550899+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.551242+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.555315+0000 osd.6 (osd.6) 286 : cluster 0 10.a starting backfill to osd.0 from (298'3819,307'7980] MIN to 317'11254 2023-10-09T06:48:59.559 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.555906+0000 osd.6 (osd.6) 287 : cluster 0 10.f starting backfill to osd.0 from (306'6676,314'10854] MIN to 317'11223 2023-10-09T06:48:59.560 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:58.562445+0000 osd.4 (osd.4) 273 : cluster 0 10.4 starting backfill to osd.0 from (303'5300,311'10250] MIN to 317'11365 2023-10-09T06:48:59.560 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: audit 2023-10-09T06:48:59.395807+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:59.560 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:48:59 smithi155 bash[59495]: cluster 2023-10-09T06:48:59.465843+0000 mgr.y (mgr.25723) 1338 : cluster 0 pgmap v1459: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 676 op/s; 1073/49119 objects misplaced (2.184%) 2023-10-09T06:48:59.583 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:48:59.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:48:59.687 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:59.732 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:48:59.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.546469+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.549278+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e320: 8 total, 8 up, 4 in 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.550807+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.550899+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.551242+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.555315+0000 osd.6 (osd.6) 286 : cluster 0 10.a starting backfill to osd.0 from (298'3819,307'7980] MIN to 317'11254 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.555906+0000 osd.6 (osd.6) 287 : cluster 0 10.f starting backfill to osd.0 from (306'6676,314'10854] MIN to 317'11223 2023-10-09T06:48:59.844 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:58.562445+0000 osd.4 (osd.4) 273 : cluster 0 10.4 starting backfill to osd.0 from (303'5300,311'10250] MIN to 317'11365 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: audit 2023-10-09T06:48:59.395807+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[77299]: cluster 2023-10-09T06:48:59.465843+0000 mgr.y (mgr.25723) 1338 : cluster 0 pgmap v1459: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 676 op/s; 1073/49119 objects misplaced (2.184%) 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.546469+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.549278+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e320: 8 total, 8 up, 4 in 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.550807+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.550899+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.845 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.551242+0000 osd.4 (osd.4) 272 : cluster 3 failed to encode map e320 with expected crc 2023-10-09T06:48:59.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.555315+0000 osd.6 (osd.6) 286 : cluster 0 10.a starting backfill to osd.0 from (298'3819,307'7980] MIN to 317'11254 2023-10-09T06:48:59.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.555906+0000 osd.6 (osd.6) 287 : cluster 0 10.f starting backfill to osd.0 from (306'6676,314'10854] MIN to 317'11223 2023-10-09T06:48:59.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:58.562445+0000 osd.4 (osd.4) 273 : cluster 0 10.4 starting backfill to osd.0 from (303'5300,311'10250] MIN to 317'11365 2023-10-09T06:48:59.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: audit 2023-10-09T06:48:59.395807+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:48:59.846 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:48:59 smithi125 bash[80002]: cluster 2023-10-09T06:48:59.465843+0000 mgr.y (mgr.25723) 1338 : cluster 0 pgmap v1459: 89 pgs: 6 active+clean+remapped, 83 active+clean; 1.0 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 676 op/s; 1073/49119 objects misplaced (2.184%) 2023-10-09T06:48:59.859 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:48:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:48:59.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:59.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:48:59.999 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:48:59.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:00.100 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:00.133 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:00.236 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:00.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:00.386 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:00.409 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:00.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:00.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:49:00.514 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:00.575 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:00.681 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:00.713 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:00.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:00.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:00.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:00.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:00.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:01.057 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:49:01.084 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:49:01.085 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:01.108 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:49:01.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:01.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:01.137 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:49:01.166 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:49:01.193 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:49:01.228 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.233 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:49:01.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:01.271 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.276 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:49:01.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:01.485 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:01.493 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:01 smithi125 bash[77299]: audit 2023-10-09T06:49:01.022323+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.125:0/1979492990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:01.494 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:01] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.33.4" 2023-10-09T06:49:01.494 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:01 smithi125 bash[80002]: audit 2023-10-09T06:49:01.022323+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.125:0/1979492990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:01.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:01.614 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.649 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:01.752 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:01.763 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:01 smithi155 bash[59495]: audit 2023-10-09T06:49:01.022323+0000 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.125:0/1979492990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:01.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:01.891 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:01.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:01.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:02.025 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:02.053 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:02.157 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:02.187 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:02.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:02.294 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:02.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:02.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:02.432 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:02.460 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:02.489 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:49:02.564 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:02.577 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:02 smithi155 bash[59495]: cluster 2023-10-09T06:49:01.466732+0000 mgr.y (mgr.25723) 1339 : cluster 0 pgmap v1460: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 411 op/s; 3490/41715 objects misplaced (8.366%); 459 KiB/s, 7 objects/s recovering 2023-10-09T06:49:02.610 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:02.714 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:02.728 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:02 smithi125 bash[80002]: cluster 2023-10-09T06:49:01.466732+0000 mgr.y (mgr.25723) 1339 : cluster 0 pgmap v1460: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 411 op/s; 3490/41715 objects misplaced (8.366%); 459 KiB/s, 7 objects/s recovering 2023-10-09T06:49:02.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:02 smithi125 bash[77299]: cluster 2023-10-09T06:49:01.466732+0000 mgr.y (mgr.25723) 1339 : cluster 0 pgmap v1460: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 411 op/s; 3490/41715 objects misplaced (8.366%); 459 KiB/s, 7 objects/s recovering 2023-10-09T06:49:02.750 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:02.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:02.887 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:02.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:02.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:02.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:03.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:03.135 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:03.174 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:03.279 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:03.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:03.339 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:03.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:03.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:03.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:03.573 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:03.684 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:03.707 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:03.810 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:03.837 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:03.940 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:03.963 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:03.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:03.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:04.068 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:04.103 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:04.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.237 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:04.340 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:04.470 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:04.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:04.601 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.629 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:04.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:04.761 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:49:04.763 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:04.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:04.866 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[77299]: cluster 2023-10-09T06:49:03.467326+0000 mgr.y (mgr.25723) 1340 : cluster 0 pgmap v1461: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 355 op/s; 3490/41715 objects misplaced (8.366%); 396 KiB/s, 6 objects/s recovering 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[77299]: audit 2023-10-09T06:49:03.619982+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[77299]: audit 2023-10-09T06:49:03.621563+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[80002]: cluster 2023-10-09T06:49:03.467326+0000 mgr.y (mgr.25723) 1340 : cluster 0 pgmap v1461: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 355 op/s; 3490/41715 objects misplaced (8.366%); 396 KiB/s, 6 objects/s recovering 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[80002]: audit 2023-10-09T06:49:03.619982+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:04.876 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:04 smithi125 bash[80002]: audit 2023-10-09T06:49:03.621563+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:04.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:04 smithi155 bash[59495]: cluster 2023-10-09T06:49:03.467326+0000 mgr.y (mgr.25723) 1340 : cluster 0 pgmap v1461: 89 pgs: 3 active+remapped+backfill_wait, 1 active+remapped+backfilling, 85 active+clean; 872 MiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 355 op/s; 3490/41715 objects misplaced (8.366%); 396 KiB/s, 6 objects/s recovering 2023-10-09T06:49:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:04 smithi155 bash[59495]: audit 2023-10-09T06:49:03.619982+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:04 smithi155 bash[59495]: audit 2023-10-09T06:49:03.621563+0000 mon.a (mon.0) 1653 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:04.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:04.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:04.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.040 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:05.142 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:05.290 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:05.316 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:05.419 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:05.448 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:05.553 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:05.700 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:05.730 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:05.837 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:05.869 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:05.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:05.974 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:06.007 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:06.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:06.109 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:06.135 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:06.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:06.238 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:06.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:06.278 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:06.371 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:06.406 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:06.509 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:06.533 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:06.636 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:06 smithi155 bash[59495]: cluster 2023-10-09T06:49:05.468214+0000 mgr.y (mgr.25723) 1341 : cluster 0 pgmap v1462: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 2425/30243 objects misplaced (8.018%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:49:06.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:06 smithi125 bash[77299]: cluster 2023-10-09T06:49:05.468214+0000 mgr.y (mgr.25723) 1341 : cluster 0 pgmap v1462: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 2425/30243 objects misplaced (8.018%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:49:06.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:06 smithi125 bash[80002]: cluster 2023-10-09T06:49:05.468214+0000 mgr.y (mgr.25723) 1341 : cluster 0 pgmap v1462: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 2425/30243 objects misplaced (8.018%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T06:49:06.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:06.660 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:06.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:06.764 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:06.799 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:06.906 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:06.935 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:06.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:06.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:07.038 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:07.062 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:07.165 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:07.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:07.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:07.294 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:07.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:07.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:07.457 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:07.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:07.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:07.610 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:07.714 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:07.750 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:07.855 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:07.878 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:07.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:07.981 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:08.006 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:08.110 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:08.133 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:08.237 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:08.270 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:08.376 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:08.398 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:08.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:08.503 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:08.557 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:08.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:08.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:08 smithi125 bash[77299]: cluster 2023-10-09T06:49:07.468648+0000 mgr.y (mgr.25723) 1342 : cluster 0 pgmap v1463: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 2425/30243 objects misplaced (8.018%); 916 KiB/s, 14 objects/s recovering 2023-10-09T06:49:08.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:08 smithi125 bash[80002]: cluster 2023-10-09T06:49:07.468648+0000 mgr.y (mgr.25723) 1342 : cluster 0 pgmap v1463: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 2425/30243 objects misplaced (8.018%); 916 KiB/s, 14 objects/s recovering 2023-10-09T06:49:08.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:08.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:08.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:08.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:08.797 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:08 smithi155 bash[59495]: cluster 2023-10-09T06:49:07.468648+0000 mgr.y (mgr.25723) 1342 : cluster 0 pgmap v1463: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 2425/30243 objects misplaced (8.018%); 916 KiB/s, 14 objects/s recovering 2023-10-09T06:49:08.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:08.921 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:08.952 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:08.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:09.064 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:09.088 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:09.191 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:09.318 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:09.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.364 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:09.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:09.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:09.597 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.626 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:09.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:09.749 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98216 10.4 10:22d265a5:::benchmark_data_smithi125_222801_object330:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.523382+0000", 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.210550938, 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1279697500000001, 2023-10-09T06:49:09.750 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98216 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.751 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523382+0000", 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523382+0000", 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.752 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523380+0000", 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6580000000000002e-06 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523388+0000", 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2544000000000001e-05 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.753 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523401+0000", 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8979999999999999e-06 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.523408+0000", 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0067794400000000003 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530187+0000", 2023-10-09T06:49:09.754 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1316e-05 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530198+0000", 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98178659899999998 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.511985+0000", 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4688999999999999e-05 2023-10-09T06:49:09.755 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512000+0000", 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1207067209999999 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.632706+0000", 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000188999 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.756 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.632895+0000", 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000222437 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.757 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.633118+0000", 2023-10-09T06:49:09.758 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010886016 2023-10-09T06:49:09.758 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644004+0000", 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00022250500000000001 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644226+0000", 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068980179999999997 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.759 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651124+0000", 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4359e-05 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651169+0000", 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2454000000000003e-05 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.760 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651221+0000", 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013100000000000001 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651352+0000", 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1279697500000001 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.761 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98219 10.4 10:2c41fec5:::benchmark_data_smithi125_222801_object333:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.526230+0000", 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.207703106, 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1254997750000002, 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.762 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98219 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526230+0000", 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.763 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526230+0000", 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526229+0000", 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2119999999999997e-06 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.764 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526234+0000", 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1460000000000001e-06 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526240+0000", 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.5439999999999999e-06 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526243+0000", 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039801790000000004 2023-10-09T06:49:09.765 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530223+0000", 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2760000000000006e-06 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530233+0000", 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98179836300000001 2023-10-09T06:49:09.766 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512031+0000", 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0889e-05 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512042+0000", 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.124785795 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.767 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.768 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.768 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.636828+0000", 2023-10-09T06:49:09.768 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7603999999999998e-05 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.636905+0000", 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6630000000000002e-05 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.769 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.636992+0000", 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0072754930000000001 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644268+0000", 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.9771000000000003e-05 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.770 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644307+0000", 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.007053625 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.771 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651361+0000", 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1477999999999998e-05 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651402+0000", 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014345699999999999 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651546+0000", 2023-10-09T06:49:09.772 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018407699999999999 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651730+0000", 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1254997750000002 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.773 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98220 10.4 10:2d42eecb:::benchmark_data_smithi125_222801_object334:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.526253+0000", 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.207679926, 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.138903864, 2023-10-09T06:49:09.774 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98220 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.775 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526253+0000", 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526253+0000", 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:49:09.776 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526253+0000", 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.384e-06 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.777 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526255+0000", 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0720000000000001e-06 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526259+0000", 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0509999999999998e-06 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.526263+0000", 2023-10-09T06:49:09.778 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0039933429999999999 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530257+0000", 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.2930000000000006e-06 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.530266+0000", 2023-10-09T06:49:09.779 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.98180236200000004 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512068+0000", 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.9329999999999996e-06 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512078+0000", 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.128301024 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.780 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.640379+0000", 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018700999999999999 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.640566+0000", 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020385999999999999 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.781 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.640770+0000", 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0035780920000000002 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644348+0000", 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0070187440000000004 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.782 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651367+0000", 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011861138 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.663228+0000", 2023-10-09T06:49:09.786 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0017250029999999999 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.664953+0000", 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2133999999999998e-05 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665015+0000", 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014176600000000001 2023-10-09T06:49:09.787 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665157+0000", 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.138903864 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98227 10.4 10:27e2ff6d:::benchmark_data_smithi125_222801_object341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.535310+0000", 2023-10-09T06:49:09.788 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.19862310000001, 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.130285577, 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98227 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.789 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535310+0000", 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535310+0000", 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.790 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535308+0000", 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5000000000000002e-06 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535316+0000", 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2466999999999999e-05 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.791 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535328+0000", 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6240000000000002e-06 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535336+0000", 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000339234 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.792 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535675+0000", 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5977e-05 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.535691+0000", 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97641577400000001 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512107+0000", 2023-10-09T06:49:09.793 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1025e-05 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512118+0000", 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.132249313 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644367+0000", 2023-10-09T06:49:09.794 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.5056000000000006e-05 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644462+0000", 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4990999999999999e-05 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.644557+0000", 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066034099999999997 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.795 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651161+0000", 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012015324000000001 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.663176+0000", 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0021422530000000002 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.796 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665318+0000", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9223e-05 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665367+0000", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2743000000000005e-05 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665440+0000", 2023-10-09T06:49:09.797 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015553499999999999 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665596+0000", 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.130285577 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.798 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98231 10.4 10:28ee891e:::benchmark_data_smithi125_222801_object345:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.540299+0000", 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.193633848, 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1256003030000001, 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98231 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.799 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540299+0000", 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540299+0000", 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.800 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540298+0000", 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3250000000000002e-06 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540305+0000", 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2204999999999999e-05 2023-10-09T06:49:09.801 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540318+0000", 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9569999999999996e-06 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540325+0000", 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1894999999999998e-05 2023-10-09T06:49:09.802 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540366+0000", 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0672e-05 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.540377+0000", 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.97176987199999998 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.803 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512147+0000", 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1351e-05 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512158+0000", 2023-10-09T06:49:09.804 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.135775639 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.647934+0000", 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.6708999999999999e-05 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.648031+0000", 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5699999999999996e-05 2023-10-09T06:49:09.805 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.648116+0000", 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0036610039999999998 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.806 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651777+0000", 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.011406743 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.663184+0000", 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025078119999999999 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665692+0000", 2023-10-09T06:49:09.807 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6523000000000003e-05 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665728+0000", 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4919000000000001e-05 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.808 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665773+0000", 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000126363 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665900+0000", 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1256003030000001 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.809 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98242 10.4 10:238bc0ec:::benchmark_data_smithi125_222801_object356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.558310+0000", 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.175623179, 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1110443229999998, 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.810 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98242 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558310+0000", 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.811 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558310+0000", 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558309+0000", 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2590000000000003e-06 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.812 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558316+0000", 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1629e-05 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558328+0000", 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2800000000000003e-06 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.813 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558336+0000", 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6836999999999996e-05 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558413+0000", 2023-10-09T06:49:09.814 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6361000000000001e-05 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.558429+0000", 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95375791499999996 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512187+0000", 2023-10-09T06:49:09.815 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0057999999999999e-05 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512197+0000", 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.139603744 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651801+0000", 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012633499999999999 2023-10-09T06:49:09.816 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.651927+0000", 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015289200000000001 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.652080+0000", 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012798835999999999 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.817 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.664879+0000", 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033135499999999997 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665210+0000", 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00072737099999999999 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.818 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665938+0000", 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032330089999999998 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669171+0000", 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3057999999999999e-05 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669224+0000", 2023-10-09T06:49:09.819 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013080700000000001 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669354+0000", 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.1110443229999998 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.820 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98244 10.4 10:2a239f13:::benchmark_data_smithi125_222801_object358:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.559919+0000", 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.17401390400001, 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.109647523, 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98244 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.821 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559919+0000", 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559919+0000", 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.822 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559918+0000", 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7819999999999996e-06 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559925+0000", 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2119e-05 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.823 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559937+0000", 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.4999999999999996e-06 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.559944+0000", 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1264e-05 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.824 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.560025+0000", 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2874000000000001e-05 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.560038+0000", 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95221024600000004 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512248+0000", 2023-10-09T06:49:09.825 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0900999999999999e-05 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512259+0000", 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1428235 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.655082+0000", 2023-10-09T06:49:09.826 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121149 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.655203+0000", 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000151533 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.655355+0000", 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0099740179999999994 2023-10-09T06:49:09.827 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665329+0000", 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00031021299999999999 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665639+0000", 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00033392799999999999 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.828 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665973+0000", 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034209940000000001 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669394+0000", 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0281999999999997e-05 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.829 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669444+0000", 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012241199999999999 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669567+0000", 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.109647523 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.830 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98266 10.4 10:2cd9c0de:::benchmark_data_smithi125_222801_object380:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.582295+0000", 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.15163864500001, 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.103923403, 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.831 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98266 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582295+0000", 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582295+0000", 2023-10-09T06:49:09.832 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582293+0000", 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7970000000000001e-06 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582301+0000", 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3648e-05 2023-10-09T06:49:09.833 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582314+0000", 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.187e-06 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582322+0000", 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5658e-05 2023-10-09T06:49:09.834 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582397+0000", 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5478e-05 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.582413+0000", 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.92989771200000004 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.835 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512310+0000", 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0753e-05 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512321+0000", 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1536733429999999 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.836 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.665994+0000", 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000123941 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.666118+0000", 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014537799999999999 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.666264+0000", 2023-10-09T06:49:09.837 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013022351 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679286+0000", 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3757999999999997e-05 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679330+0000", 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6867999999999998e-05 2023-10-09T06:49:09.838 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679387+0000", 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006643581 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.686030+0000", 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4364000000000003e-05 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.839 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.686085+0000", 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000133277 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.686218+0000", 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.103923403 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.840 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98289 10.4 10:266e9ebd:::benchmark_data_smithi125_222801_object403:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.619569+0000", 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.114364438, 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0730322829999999, 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.841 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98289 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619569+0000", 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.842 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619569+0000", 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619568+0000", 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.57e-06 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619573+0000", 2023-10-09T06:49:09.843 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0172e-05 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619583+0000", 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9619999999999996e-06 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619590+0000", 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2626999999999995e-05 2023-10-09T06:49:09.844 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619653+0000", 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7391e-05 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.619670+0000", 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.892681752 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.845 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512352+0000", 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0794e-05 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512363+0000", 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1572305220000001 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.846 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.669593+0000", 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00047468400000000001 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.670068+0000", 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016429199999999999 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.847 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.670232+0000", 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0091055430000000007 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679338+0000", 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5031000000000006e-05 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679423+0000", 2023-10-09T06:49:09.848 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01289209 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.692315+0000", 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010689899999999999 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.692422+0000", 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7394999999999997e-05 2023-10-09T06:49:09.849 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.692459+0000", 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014176799999999999 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.692601+0000", 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0730322829999999 2023-10-09T06:49:09.850 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98291 10.4 10:274a5e8a:::benchmark_data_smithi125_222801_object405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.624947+0000", 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 122.108986289, 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0744618340000001, 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.851 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98291 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624947+0000", 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.852 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624947+0000", 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624946+0000", 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.19e-06 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.853 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624952+0000", 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2424e-05 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624964+0000", 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2190000000000001e-06 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.854 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.624970+0000", 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6307e-05 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.625047+0000", 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6188000000000002e-05 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.855 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.625063+0000", 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.88751327499999999 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512576+0000", 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6174000000000001e-05 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.512592+0000", 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.160354023 2023-10-09T06:49:09.856 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.672946+0000", 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012286999999999999 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.673069+0000", 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015717699999999999 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.857 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.673226+0000", 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0061216930000000001 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679348+0000", 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000112853 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.858 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.679461+0000", 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012888474 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.692349+0000", 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0068586239999999998 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.859 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.699208+0000", 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0406999999999999e-05 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.699268+0000", 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014030499999999999 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.860 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:09.699409+0000", 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0744618340000001 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162629 10.4 10:22b8b508:::benchmark_data_smithi125_222801_object64743:head [delete] snapc 0=[] ondisk+write+known_if_redirected e311)", 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.158014+0000", 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.575919351000003, 2023-10-09T06:49:09.861 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3792078430000001, 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162629 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.862 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158014+0000", 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158014+0000", 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158013+0000", 2023-10-09T06:49:09.863 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.1439999999999992e-06 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158022+0000", 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.3629999999999999e-05 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158045+0000", 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.6209999999999998e-06 2023-10-09T06:49:09.864 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158052+0000", 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0989000000000003e-05 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158103+0000", 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.5385999999999999e-05 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.865 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158188+0000", 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8143e-05 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.158277+0000", 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.024573714999999999 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.866 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.182850+0000", 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0274e-05 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.182861+0000", 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.354283313 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.867 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.537144+0000", 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7875999999999998e-05 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.537222+0000", 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3792078430000001 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.868 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162637 10.4 10:29fd5957:::benchmark_data_smithi125_222801_object64751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e311)", 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.159818+0000", 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.574115026999998, 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3868314850000001, 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.869 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162637 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159818+0000", 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.870 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159818+0000", 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159817+0000", 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6569999999999996e-06 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159822+0000", 2023-10-09T06:49:09.871 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.985e-06 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159829+0000", 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6090000000000003e-06 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.159834+0000", 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015004457000000001 2023-10-09T06:49:09.872 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.174838+0000", 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014967 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.174988+0000", 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000169631 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.873 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6,7", 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.175157+0000", 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0084340130000000006 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.183591+0000", 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.87e-06 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.874 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.183596+0000", 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3542652989999999 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.537862+0000", 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00020441799999999999 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.875 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for missing object", 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.538066+0000", 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0084667800000000001 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546533+0000", 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011700400000000001 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.876 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546650+0000", 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3868314850000001 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162641 10.4 10:24256d0d:::benchmark_data_smithi125_222801_object64755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.170622+0000", 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.563310942999998, 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.376819397, 2023-10-09T06:49:09.877 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162641 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170622+0000", 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.878 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170622+0000", 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170621+0000", 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1840000000000002e-06 2023-10-09T06:49:09.879 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170628+0000", 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4837999999999999e-05 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170643+0000", 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.1500000000000002e-06 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.880 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.170650+0000", 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.013221139 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.183871+0000", 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6699999999999994e-06 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.881 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.183879+0000", 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3542826450000001 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.538161+0000", 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000167897 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.882 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.538329+0000", 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000156193 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.538486+0000", 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0079739630000000006 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.883 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546460+0000", 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00024836900000000002 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546708+0000", 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00023793100000000001 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.884 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546946+0000", 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035652299999999997 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547302+0000", 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013934599999999999 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.885 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547442+0000", 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.376819397 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162669 10.4 10:2d2b5071:::benchmark_data_smithi125_222801_object64783:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.219761+0000", 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.514172041000002, 2023-10-09T06:49:09.886 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.327991041, 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162669 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.887 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219761+0000", 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219761+0000", 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.888 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219761+0000", 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.2289999999999997e-06 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219765+0000", 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.4199999999999999e-05 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219789+0000", 2023-10-09T06:49:09.889 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0519999999999998e-06 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219795+0000", 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.031268059000000001 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.251063+0000", 2023-10-09T06:49:09.890 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7600000000000002e-06 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.251071+0000", 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2913237049999999 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.542395+0000", 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000166909 2023-10-09T06:49:09.891 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.542561+0000", 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000120406 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.542682+0000", 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0041126469999999997 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.892 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546795+0000", 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3481e-05 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.546808+0000", 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00071167799999999999 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.893 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547520+0000", 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.6896000000000004e-05 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547597+0000", 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015568799999999999 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.894 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547752+0000", 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.327991041 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162670 10.4 10:243f7876:::benchmark_data_smithi125_222801_object64784:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.219802+0000", 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.514131536000001, 2023-10-09T06:49:09.895 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.345948269, 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162670 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.896 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219802+0000", 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219802+0000", 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.897 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219801+0000", 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.907e-06 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219805+0000", 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0460000000000002e-06 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.898 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219811+0000", 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8809999999999998e-06 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.219815+0000", 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.036346748999999998 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.899 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256162+0000", 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.8074e-05 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256180+0000", 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.291802345 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.900 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.547982+0000", 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000209237 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.548191+0000", 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00015581699999999999 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.548347+0000", 2023-10-09T06:49:09.901 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01085799 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.559205+0000", 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000119535 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.559325+0000", 2023-10-09T06:49:09.902 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006313768 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565638+0000", 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1050000000000007e-05 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565700+0000", 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0473000000000003e-05 2023-10-09T06:49:09.903 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565750+0000", 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.345948269 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162672 10.4 10:27bb8d95:::benchmark_data_smithi125_222801_object64786:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.904 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.220382+0000", 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.513551219999997, 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3519368460000001, 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162672 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.905 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220382+0000", 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220382+0000", 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.906 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220381+0000", 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8780000000000003e-06 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220387+0000", 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0410000000000003e-06 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.907 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220396+0000", 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.772e-06 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.220402+0000", 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.035797412000000001 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256199+0000", 2023-10-09T06:49:09.908 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5350000000000001e-06 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256206+0000", 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2988939930000001 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.555100+0000", 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013665699999999999 2023-10-09T06:49:09.909 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.555236+0000", 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018278399999999999 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.555419+0000", 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010383921000000001 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.910 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565803+0000", 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032145239999999999 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.569017+0000", 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0032041109999999999 2023-10-09T06:49:09.911 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572222+0000", 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3939999999999997e-05 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572256+0000", 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3369999999999998e-05 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.912 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572319+0000", 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3519368460000001 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162683 10.4 10:206c7e13:::benchmark_data_smithi125_222801_object64797:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.236282+0000", 2023-10-09T06:49:09.913 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.497651521000002, 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.341238578, 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162683 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.914 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236282+0000", 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236282+0000", 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.915 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236280+0000", 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5660000000000001e-06 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236288+0000", 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.0038e-05 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.916 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236308+0000", 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8549999999999996e-06 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.236315+0000", 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.019908058999999999 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.917 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256223+0000", 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3140000000000003e-06 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256228+0000", 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3031215729999999 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.559350+0000", 2023-10-09T06:49:09.918 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000100313 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.559450+0000", 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012642799999999999 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.559576+0000", 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012609095000000001 2023-10-09T06:49:09.919 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572185+0000", 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016804600000000001 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572353+0000", 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050879699999999998 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.920 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.577441+0000", 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.4712000000000002e-05 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.577476+0000", 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4183e-05 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.921 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.577520+0000", 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.341238578 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162687 10.4 10:2f1abf89:::benchmark_data_smithi125_222801_object64801:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.238943+0000", 2023-10-09T06:49:09.922 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.494990604000002, 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3455824999999999, 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162687 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.923 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238943+0000", 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238943+0000", 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.924 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238942+0000", 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5679999999999996e-06 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238948+0000", 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3784000000000001e-05 2023-10-09T06:49:09.925 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238962+0000", 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8680000000000001e-06 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.238969+0000", 2023-10-09T06:49:09.926 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.017275954 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256245+0000", 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.3240000000000002e-06 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256250+0000", 2023-10-09T06:49:09.927 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.306429635 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.562680+0000", 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109715 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.562789+0000", 2023-10-09T06:49:09.928 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013520599999999999 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.562925+0000", 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0092671679999999992 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.572192+0000", 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028767490000000001 2023-10-09T06:49:09.929 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.575068+0000", 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0093697750000000003 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584438+0000", 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5682000000000001e-05 2023-10-09T06:49:09.930 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584484+0000", 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1202000000000003e-05 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584525+0000", 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3455824999999999 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.931 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162693 10.4 10:2a80550a:::benchmark_data_smithi125_222801_object64807:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.249544+0000", 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.484389542999999, 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.335165446, 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.932 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162693 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249544+0000", 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.933 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249544+0000", 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249542+0000", 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3180000000000001e-06 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.934 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249549+0000", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3869999999999999e-05 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249563+0000", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0240000000000003e-06 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.249569+0000", 2023-10-09T06:49:09.935 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006700124 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256269+0000", 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.186e-06 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.936 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.256274+0000", 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.309552861 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565827+0000", 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013434699999999999 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.565961+0000", 2023-10-09T06:49:09.937 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00018670399999999999 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.566148+0000", 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.018215392 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584363+0000", 2023-10-09T06:49:09.938 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3212e-05 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584377+0000", 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000257619 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584634+0000", 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.701e-05 2023-10-09T06:49:09.939 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584671+0000", 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7836000000000002e-05 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584709+0000", 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.335165446 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.940 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:162695 10.4 10:2360eabb:::benchmark_data_smithi125_222801_object64809:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:31.252629+0000", 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 38.481304064, 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.341738246, 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:49:09.941 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 162695 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252629+0000", 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.942 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252629+0000", 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252628+0000", 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2820000000000002e-06 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.943 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252634+0000", 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8009999999999998e-06 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252644+0000", 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.1070000000000001e-06 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:49:09.944 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.252650+0000", 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0069266370000000002 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.259577+0000", 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.943e-06 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:49:09.945 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:31.259585+0000", 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3094592089999999 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.569044+0000", 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010624999999999999 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:49:09.946 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.569150+0000", 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000125432 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.569276+0000", 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.015109819 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:49:09.947 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584386+0000", 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000193463 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.584579+0000", 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0096977520000000005 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.594277+0000", 2023-10-09T06:49:09.948 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9753000000000003e-05 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.594327+0000", 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0806999999999997e-05 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:32.594367+0000", 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.341738246 2023-10-09T06:49:09.949 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.950 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.950 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.950 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:49:09.950 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:49:09.950 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:49:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:09.951 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:09.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:10.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:10.079 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:10.183 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:10.218 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:10.322 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:10.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:10 smithi155 bash[59495]: cluster 2023-10-09T06:49:09.469137+0000 mgr.y (mgr.25723) 1343 : cluster 0 pgmap v1464: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 575 op/s; 2425/30243 objects misplaced (8.018%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:49:10.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:10.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:10.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:10.494 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:10.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:10.625 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:10.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:10 smithi125 bash[77299]: cluster 2023-10-09T06:49:09.469137+0000 mgr.y (mgr.25723) 1343 : cluster 0 pgmap v1464: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 575 op/s; 2425/30243 objects misplaced (8.018%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:49:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:10 smithi125 bash[80002]: cluster 2023-10-09T06:49:09.469137+0000 mgr.y (mgr.25723) 1343 : cluster 0 pgmap v1464: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 633 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 575 op/s; 2425/30243 objects misplaced (8.018%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:49:10.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:10.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:10.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:10.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:10.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:10.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:10.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:11.019 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:11.123 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:11.141 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:11.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:11.244 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:11.264 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:11.266 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:11] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.33.4" 2023-10-09T06:49:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:11.368 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:11.393 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:11.496 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:11.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:11.628 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:11.657 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:11.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:11.721 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:49:11.762 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:11.786 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:11.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:11.889 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:11.935 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:11.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:11.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:12.041 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.063 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:12.216 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:12.250 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:12.352 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.380 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:12.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:12.509 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:12.528 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:12 smithi155 bash[59495]: cluster 2023-10-09T06:49:11.469783+0000 mgr.y (mgr.25723) 1344 : cluster 0 pgmap v1465: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 1475/19224 objects misplaced (7.673%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:49:12.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:12.638 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:12.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:12 smithi125 bash[80002]: cluster 2023-10-09T06:49:11.469783+0000 mgr.y (mgr.25723) 1344 : cluster 0 pgmap v1465: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 1475/19224 objects misplaced (7.673%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:49:12.673 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:12 smithi125 bash[77299]: cluster 2023-10-09T06:49:11.469783+0000 mgr.y (mgr.25723) 1344 : cluster 0 pgmap v1465: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 1475/19224 objects misplaced (7.673%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T06:49:12.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.764 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:12.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.892 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:12.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:12.995 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:13.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:13.120 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:13.149 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:13.255 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:13.281 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:13.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:13.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:13.518 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:13.546 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:13.650 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:13.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:13.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:13.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:13.813 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:13.920 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:13.943 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:13.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:14.046 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:14.068 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:14.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:14.173 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:14.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:14.327 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:14.349 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:14.454 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:14.479 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:14.533 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:14 smithi155 bash[59495]: cluster 2023-10-09T06:49:13.470314+0000 mgr.y (mgr.25723) 1345 : cluster 0 pgmap v1466: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1475/19224 objects misplaced (7.673%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:14.582 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:14.618 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:14.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:14.729 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:14 smithi125 bash[77299]: cluster 2023-10-09T06:49:13.470314+0000 mgr.y (mgr.25723) 1345 : cluster 0 pgmap v1466: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1475/19224 objects misplaced (7.673%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:14.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:14 smithi125 bash[80002]: cluster 2023-10-09T06:49:13.470314+0000 mgr.y (mgr.25723) 1345 : cluster 0 pgmap v1466: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 403 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1475/19224 objects misplaced (7.673%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:14.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:14.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:14.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:14.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:14.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:14.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:15.003 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:15.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:15.156 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:15.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:15.263 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:15.293 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:15.398 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:15.418 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:15.521 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:15.545 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:15.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:15.674 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:15.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:15.776 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:15.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:15.900 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:15.936 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:15.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:16.039 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:16.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:16.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:16.170 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:16.196 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:16.299 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:16.347 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:16.453 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:16.473 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:16.575 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:16.578 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:16.587 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:16 smithi155 bash[59495]: cluster 2023-10-09T06:49:15.471017+0000 mgr.y (mgr.25723) 1346 : cluster 0 pgmap v1467: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 946 op/s; 525/7632 objects misplaced (6.879%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:49:16.620 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:16.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:16.737 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:16 smithi125 bash[77299]: cluster 2023-10-09T06:49:15.471017+0000 mgr.y (mgr.25723) 1346 : cluster 0 pgmap v1467: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 946 op/s; 525/7632 objects misplaced (6.879%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:49:16.738 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:16 smithi125 bash[80002]: cluster 2023-10-09T06:49:15.471017+0000 mgr.y (mgr.25723) 1346 : cluster 0 pgmap v1467: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 946 op/s; 525/7632 objects misplaced (6.879%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:49:16.752 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:16.855 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:16.875 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:16.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:16.978 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:17.001 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:17.105 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:17.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:17.234 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:17.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:17.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:17.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:17.382 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:17.485 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:17.520 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:17.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:17.626 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:17.656 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:17.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:17.760 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:17.789 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:17.895 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:17.923 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:17.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:17.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:18.026 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:18.055 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:18.158 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:18.184 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:18.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:18.308 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:18.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:18.411 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:18.441 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:18.544 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:18.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[77299]: cluster 2023-10-09T06:49:17.471603+0000 mgr.y (mgr.25723) 1347 : cluster 0 pgmap v1468: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 525/7632 objects misplaced (6.879%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:18.545 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[77299]: cluster 2023-10-09T06:49:17.545172+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e321: 8 total, 8 up, 4 in 2023-10-09T06:49:18.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[77299]: cluster 2023-10-09T06:49:17.545995+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[77299]: cluster 2023-10-09T06:49:17.547333+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.546 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[77299]: cluster 2023-10-09T06:49:17.549589+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.546 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[80002]: cluster 2023-10-09T06:49:17.471603+0000 mgr.y (mgr.25723) 1347 : cluster 0 pgmap v1468: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 525/7632 objects misplaced (6.879%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:18.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[80002]: cluster 2023-10-09T06:49:17.545172+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e321: 8 total, 8 up, 4 in 2023-10-09T06:49:18.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[80002]: cluster 2023-10-09T06:49:17.545995+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[80002]: cluster 2023-10-09T06:49:17.547333+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:18 smithi125 bash[80002]: cluster 2023-10-09T06:49:17.549589+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:18.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:18.682 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:18 smithi155 bash[59495]: cluster 2023-10-09T06:49:17.471603+0000 mgr.y (mgr.25723) 1347 : cluster 0 pgmap v1468: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 525/7632 objects misplaced (6.879%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:18.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:18 smithi155 bash[59495]: cluster 2023-10-09T06:49:17.545172+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e321: 8 total, 8 up, 4 in 2023-10-09T06:49:18.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:18 smithi155 bash[59495]: cluster 2023-10-09T06:49:17.545995+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:18 smithi155 bash[59495]: cluster 2023-10-09T06:49:17.547333+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.683 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:18 smithi155 bash[59495]: cluster 2023-10-09T06:49:17.549589+0000 osd.4 (osd.4) 274 : cluster 3 failed to encode map e321 with expected crc 2023-10-09T06:49:18.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:18.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:18.806 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:18.831 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:18.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:18.934 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:18.954 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:18.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:19.057 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:19.084 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:19.187 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:19.219 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:19.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:19.323 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:19.344 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:19.447 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:19.477 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:19.558 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:49:19.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:19.599 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:18.547006+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:18.548000+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e322: 8 total, 8 up, 4 in 2023-10-09T06:49:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:18.548407+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:18.550652+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:18.552989+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: audit 2023-10-09T06:49:18.611879+0000 mon.a (mon.0) 1656 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: audit 2023-10-09T06:49:18.927656+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.125:0/366151041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: audit 2023-10-09T06:49:18.927995+0000 mon.a (mon.0) 1657 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:19 smithi155 bash[59495]: cluster 2023-10-09T06:49:19.472060+0000 mgr.y (mgr.25723) 1348 : cluster 0 pgmap v1471: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 525/7632 objects misplaced (6.879%); 792 KiB/s, 12 objects/s recovering 2023-10-09T06:49:19.703 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:19.731 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:19.837 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:18.547006+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:18.548000+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e322: 8 total, 8 up, 4 in 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:18.548407+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:18.550652+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:18.552989+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: audit 2023-10-09T06:49:18.611879+0000 mon.a (mon.0) 1656 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: audit 2023-10-09T06:49:18.927656+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.125:0/366151041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.847 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: audit 2023-10-09T06:49:18.927995+0000 mon.a (mon.0) 1657 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[77299]: cluster 2023-10-09T06:49:19.472060+0000 mgr.y (mgr.25723) 1348 : cluster 0 pgmap v1471: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 525/7632 objects misplaced (6.879%); 792 KiB/s, 12 objects/s recovering 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:18.547006+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:18.548000+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e322: 8 total, 8 up, 4 in 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:18.548407+0000 osd.4 (osd.4) 275 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:18.550652+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:18.552989+0000 osd.4 (osd.4) 276 : cluster 3 failed to encode map e322 with expected crc 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: audit 2023-10-09T06:49:18.611879+0000 mon.a (mon.0) 1656 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:19.848 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: audit 2023-10-09T06:49:18.927656+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.125:0/366151041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: audit 2023-10-09T06:49:18.927995+0000 mon.a (mon.0) 1657 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:49:19.849 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:19 smithi125 bash[80002]: cluster 2023-10-09T06:49:19.472060+0000 mgr.y (mgr.25723) 1348 : cluster 0 pgmap v1471: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 162 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 525/7632 objects misplaced (6.879%); 792 KiB/s, 12 objects/s recovering 2023-10-09T06:49:19.863 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:19.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:19.994 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:19.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:20.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.123 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:20.226 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:20.251 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:20.354 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:20.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:20.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:20.486 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:20.507 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:20.577 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:49:20.611 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.637 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:20.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: cluster 2023-10-09T06:49:19.550242+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: audit 2023-10-09T06:49:19.551448+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: cluster 2023-10-09T06:49:19.551509+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e323: 8 total, 8 up, 4 in 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: cluster 2023-10-09T06:49:19.553073+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: audit 2023-10-09T06:49:20.038864+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.125:0/710969161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.754 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[77299]: audit 2023-10-09T06:49:20.039169+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: cluster 2023-10-09T06:49:19.550242+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: audit 2023-10-09T06:49:19.551448+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: cluster 2023-10-09T06:49:19.551509+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e323: 8 total, 8 up, 4 in 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: cluster 2023-10-09T06:49:19.553073+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: audit 2023-10-09T06:49:20.038864+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.125:0/710969161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.755 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:20 smithi125 bash[80002]: audit 2023-10-09T06:49:20.039169+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.767 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:20.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: cluster 2023-10-09T06:49:19.550242+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: audit 2023-10-09T06:49:19.551448+0000 mon.a (mon.0) 1658 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: cluster 2023-10-09T06:49:19.551509+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e323: 8 total, 8 up, 4 in 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: cluster 2023-10-09T06:49:19.553073+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e323 with expected crc 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: audit 2023-10-09T06:49:20.038864+0000 mon.c (mon.2) 134 : audit 1 from='client.? 172.21.15.125:0/710969161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.879 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:20 smithi155 bash[59495]: audit 2023-10-09T06:49:20.039169+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:49:20.897 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:20.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:21.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:21.027 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:21.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:21.265 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:21.289 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:21.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:21.393 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:21.421 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:21.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:21.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:21.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:21.562 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:21] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-10-09T06:49:21.656 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.556315+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:21.663 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.565170+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.566147+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: audit 2023-10-09T06:49:20.566197+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.566284+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e324: 8 total, 8 up, 4 in 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.567600+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.567782+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:20.569727+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.664 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:21 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.472522+0000 mgr.y (mgr.25723) 1349 : cluster 0 pgmap v1474: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:21.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:21.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.556315+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.565170+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.566147+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: audit 2023-10-09T06:49:20.566197+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:49:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.566284+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e324: 8 total, 8 up, 4 in 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.567600+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.567782+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:20.569727+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.472522+0000 mgr.y (mgr.25723) 1349 : cluster 0 pgmap v1474: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.556315+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:21.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.565170+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.566147+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: audit 2023-10-09T06:49:20.566197+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.566284+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e324: 8 total, 8 up, 4 in 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.567600+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.567782+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:20.569727+0000 osd.4 (osd.4) 277 : cluster 3 failed to encode map e324 with expected crc 2023-10-09T06:49:21.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:21 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.472522+0000 mgr.y (mgr.25723) 1349 : cluster 0 pgmap v1474: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:21.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:21.940 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:21.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:22.044 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:22.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:22.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:22.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:22.297 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Removed 97883 objects 2023-10-09T06:49:22.298 INFO:tasks.radosbench.radosbench.0.smithi125.stdout:Clean up completed and total clean up time :135.146 2023-10-09T06:49:22.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:22.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:22.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:22.610 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:22.635 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:22.738 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.573367+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.574440+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e325: 8 total, 8 up, 4 in 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.575799+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.576463+0000 osd.0 (osd.0) 21 : cluster 0 10.b starting backfill to osd.4 from (303'5900,315'10892] MIN to 322'12186 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.577903+0000 osd.6 (osd.6) 298 : cluster 0 10.0 starting backfill to osd.4 from (307'7811,320'12157] MIN to 322'12169 2023-10-09T06:49:22.750 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:22 smithi155 bash[59495]: cluster 2023-10-09T06:49:21.586411+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:22.814 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.573367+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.814 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.574440+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e325: 8 total, 8 up, 4 in 2023-10-09T06:49:22.814 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.575799+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.814 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.576463+0000 osd.0 (osd.0) 21 : cluster 0 10.b starting backfill to osd.4 from (303'5900,315'10892] MIN to 322'12186 2023-10-09T06:49:22.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.577903+0000 osd.6 (osd.6) 298 : cluster 0 10.0 starting backfill to osd.4 from (307'7811,320'12157] MIN to 322'12169 2023-10-09T06:49:22.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[77299]: cluster 2023-10-09T06:49:21.586411+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:22.866 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.895 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:22.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:23.000 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:23.026 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:23.130 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.573367+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:23.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.574440+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e325: 8 total, 8 up, 4 in 2023-10-09T06:49:23.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.575799+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:23.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.576463+0000 osd.0 (osd.0) 21 : cluster 0 10.b starting backfill to osd.4 from (303'5900,315'10892] MIN to 322'12186 2023-10-09T06:49:23.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.577903+0000 osd.6 (osd.6) 298 : cluster 0 10.0 starting backfill to osd.4 from (307'7811,320'12157] MIN to 322'12169 2023-10-09T06:49:23.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:22 smithi125 bash[80002]: cluster 2023-10-09T06:49:21.586411+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e325 with expected crc 2023-10-09T06:49:23.163 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:23.266 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:23.300 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:23.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:23.341 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:49:23.342 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:49:23.342 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T06:49:23.342 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:49:23.404 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:23.435 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:23.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:49:23.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.572 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:23.577 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:49:23.602 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:49:23.625 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: cluster 2023-10-09T06:49:22.584649+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: cluster 2023-10-09T06:49:22.585101+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e326: 8 total, 8 up, 4 in 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: cluster 2023-10-09T06:49:22.586587+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: cluster 2023-10-09T06:49:22.589309+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.473060+0000 mgr.y (mgr.25723) 1350 : cluster 0 pgmap v1477: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:23.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[77299]: audit 2023-10-09T06:49:23.548781+0000 mon.a (mon.0) 1666 : audit 0 from='client.? 172.21.15.125:0/3602853797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: cluster 2023-10-09T06:49:22.584649+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: cluster 2023-10-09T06:49:22.585101+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e326: 8 total, 8 up, 4 in 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: cluster 2023-10-09T06:49:22.586587+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: cluster 2023-10-09T06:49:22.589309+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.473060+0000 mgr.y (mgr.25723) 1350 : cluster 0 pgmap v1477: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:23.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:23 smithi125 bash[80002]: audit 2023-10-09T06:49:23.548781+0000 mon.a (mon.0) 1666 : audit 0 from='client.? 172.21.15.125:0/3602853797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:23.649 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:49:23.672 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:49:23.675 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.697 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:49:23.702 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:23.721 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:49:23.748 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:49:23.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.750 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.751 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:49:23.805 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:23.832 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: cluster 2023-10-09T06:49:22.584649+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: cluster 2023-10-09T06:49:22.585101+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e326: 8 total, 8 up, 4 in 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: cluster 2023-10-09T06:49:22.586587+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: cluster 2023-10-09T06:49:22.589309+0000 osd.4 (osd.4) 278 : cluster 3 failed to encode map e326 with expected crc 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.473060+0000 mgr.y (mgr.25723) 1350 : cluster 0 pgmap v1477: 89 pgs: 15 peering, 74 active+clean; 110 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 207 op/s 2023-10-09T06:49:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:23 smithi155 bash[59495]: audit 2023-10-09T06:49:23.548781+0000 mon.a (mon.0) 1666 : audit 0 from='client.? 172.21.15.125:0/3602853797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:23.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:23.966 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:23.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:24.070 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:24.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:24.208 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:24.235 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:24.338 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:24.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:24.406 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:49:24.467 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:24.496 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:24.600 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:24.628 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:24.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:24.742 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:24 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.590836+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.743 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:24 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.593254+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.743 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:24 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.595493+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e327: 8 total, 8 up, 4 in 2023-10-09T06:49:24.743 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:24 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.597085+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.743 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:24 smithi155 bash[59495]: cluster 2023-10-09T06:49:23.597514+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.760 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:24.857 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.590836+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.593254+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.595493+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e327: 8 total, 8 up, 4 in 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.597085+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[77299]: cluster 2023-10-09T06:49:23.597514+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.590836+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.858 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.593254+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.595493+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e327: 8 total, 8 up, 4 in 2023-10-09T06:49:24.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.597085+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.859 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:24 smithi125 bash[80002]: cluster 2023-10-09T06:49:23.597514+0000 osd.4 (osd.4) 279 : cluster 3 failed to encode map e327 with expected crc 2023-10-09T06:49:24.865 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:24.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:24.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:25.000 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.032 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:25.136 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:25.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:25.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:25.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:25.400 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:25.431 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:25.535 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:25.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:49:25.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:25.674 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:25.705 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:25.808 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.821 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:25 smithi125 bash[77299]: cluster 2023-10-09T06:49:25.474301+0000 mgr.y (mgr.25723) 1351 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:49:25.822 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:25 smithi125 bash[80002]: cluster 2023-10-09T06:49:25.474301+0000 mgr.y (mgr.25723) 1351 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:49:25.843 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:25 smithi155 bash[59495]: cluster 2023-10-09T06:49:25.474301+0000 mgr.y (mgr.25723) 1351 : cluster 0 pgmap v1479: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:49:25.947 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.983 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:25.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:26.087 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:26.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:26.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:26.257 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:26.360 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:26.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:26.491 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:26.523 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:26.627 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:26.655 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:26.758 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:26.788 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:26.860 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:49:26.892 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:26.925 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:26.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:27.029 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:27.065 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:27.168 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:27.199 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:27.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:27.303 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:27.340 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:27.426 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T06:49:27.426 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T06:49:27.444 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:27.502 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:27.606 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:27.634 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:27.738 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:27.771 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:27.874 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:27.905 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:27.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:28.009 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:28.042 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:28.145 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.177 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:28.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:28.312 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:28.388 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:28.416 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:28.447 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:28.551 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:28.564 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:28 smithi155 bash[59495]: cluster 2023-10-09T06:49:27.474914+0000 mgr.y (mgr.25723) 1352 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 280 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:49:28.581 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:28.685 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:28.697 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:28 smithi125 bash[77299]: cluster 2023-10-09T06:49:27.474914+0000 mgr.y (mgr.25723) 1352 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 280 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:49:28.697 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:28 smithi125 bash[80002]: cluster 2023-10-09T06:49:27.474914+0000 mgr.y (mgr.25723) 1352 : cluster 0 pgmap v1480: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 280 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:49:28.711 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:28.814 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.840 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:28.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.974 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:28.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:29.077 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:29.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:29.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:29.187 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:29.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:29.228 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:29.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:29.358 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:29.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:29.406 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:29.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:29.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:29.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:29.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:29.623 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:29.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:29.754 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:29.857 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:29.891 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:29.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:29.995 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:30.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:30.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:30.161 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:30.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:30.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:30.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:30.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:30 smithi155 bash[59495]: cluster 2023-10-09T06:49:29.475331+0000 mgr.y (mgr.25723) 1353 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 213 op/s; 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:49:30.399 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:30.431 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:30.535 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:30.547 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:30 smithi125 bash[80002]: cluster 2023-10-09T06:49:29.475331+0000 mgr.y (mgr.25723) 1353 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 213 op/s; 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:49:30.547 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:30 smithi125 bash[77299]: cluster 2023-10-09T06:49:29.475331+0000 mgr.y (mgr.25723) 1353 : cluster 0 pgmap v1481: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 213 op/s; 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:49:30.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:30.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:30.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:30.689 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:30.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:30.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:30.819 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:30.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:30.942 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:30.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:31.046 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:31.073 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:31.176 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:31.202 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:31.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:31.305 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.331 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:31.433 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:31.433 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:31.434 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:31.462 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:31.566 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:31.574 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:31] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.33.4" 2023-10-09T06:49:31.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:31.689 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.716 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:31.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:31.849 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:31.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:31.953 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.976 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:31.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:32.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:32.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:32.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:32.211 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:32.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:32.335 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:32.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:32.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:32.497 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:32.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:32.627 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:32.732 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:32.744 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:32 smithi155 bash[59495]: cluster 2023-10-09T06:49:31.476127+0000 mgr.y (mgr.25723) 1354 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 189 op/s; 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:32.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:32.867 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:32.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:32 smithi125 bash[77299]: cluster 2023-10-09T06:49:31.476127+0000 mgr.y (mgr.25723) 1354 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 189 op/s; 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:32.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:32 smithi125 bash[80002]: cluster 2023-10-09T06:49:31.476127+0000 mgr.y (mgr.25723) 1354 : cluster 0 pgmap v1482: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 189 op/s; 1.0 MiB/s, 16 objects/s recovering 2023-10-09T06:49:32.896 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:32.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:32.999 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:33.023 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:33.129 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:33.135 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:49:33.135 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:49:33.157 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:33.260 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:33.284 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:33.387 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:33.418 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:33.523 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:33.552 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:33.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:33.678 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:33.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:33.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:33.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:33.916 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:33.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:33.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:33.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:34.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:34.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:34.103 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:49:34.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:34.197 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:34.301 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:34.330 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:34.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.462 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:34.565 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:34.596 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:34.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:34.700 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[77299]: cluster 2023-10-09T06:49:33.476521+0000 mgr.y (mgr.25723) 1355 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 168 op/s; 915 KiB/s, 14 objects/s recovering 2023-10-09T06:49:34.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[77299]: audit 2023-10-09T06:49:33.615331+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[77299]: audit 2023-10-09T06:49:33.616513+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:34.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[77299]: audit 2023-10-09T06:49:34.426464+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.714 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[80002]: cluster 2023-10-09T06:49:33.476521+0000 mgr.y (mgr.25723) 1355 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 168 op/s; 915 KiB/s, 14 objects/s recovering 2023-10-09T06:49:34.714 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[80002]: audit 2023-10-09T06:49:33.615331+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.714 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[80002]: audit 2023-10-09T06:49:33.616513+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:34.714 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:34 smithi125 bash[80002]: audit 2023-10-09T06:49:34.426464+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.728 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:34.831 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.854 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:34.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:34 smithi155 bash[59495]: cluster 2023-10-09T06:49:33.476521+0000 mgr.y (mgr.25723) 1355 : cluster 0 pgmap v1483: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 168 op/s; 915 KiB/s, 14 objects/s recovering 2023-10-09T06:49:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:34 smithi155 bash[59495]: audit 2023-10-09T06:49:33.615331+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:34 smithi155 bash[59495]: audit 2023-10-09T06:49:33.616513+0000 mon.a (mon.0) 1669 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:34 smithi155 bash[59495]: audit 2023-10-09T06:49:34.426464+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:34.957 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:34.987 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:34.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:35.090 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:35.115 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:35.219 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:35.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:35.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:35.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:35.390 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:35.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:35.524 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:35.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:35.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:35.756 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:35.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:35.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:35.918 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:35.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:35.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:36.022 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:36.047 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:36.151 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:36.182 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:36.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:36.285 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:36.315 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:36.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:36.439 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:36.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:36.549 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:36 smithi125 bash[77299]: cluster 2023-10-09T06:49:35.477758+0000 mgr.y (mgr.25723) 1356 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 141 op/s; 770 KiB/s, 12 objects/s recovering 2023-10-09T06:49:36.574 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:36.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:36.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:36.804 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:36.817 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:36 smithi155 bash[59495]: cluster 2023-10-09T06:49:35.477758+0000 mgr.y (mgr.25723) 1356 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 141 op/s; 770 KiB/s, 12 objects/s recovering 2023-10-09T06:49:36.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:36.908 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:36 smithi125 bash[80002]: cluster 2023-10-09T06:49:35.477758+0000 mgr.y (mgr.25723) 1356 : cluster 0 pgmap v1484: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 141 op/s; 770 KiB/s, 12 objects/s recovering 2023-10-09T06:49:36.940 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:36.971 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:36.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:37.074 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:37.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:37.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:37.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:37.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:37.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:37.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:37.501 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:37.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:37.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:37.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:37.728 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:37.763 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:37.868 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:37.895 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:37.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:37.998 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.020 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:38.124 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:38.150 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:38.252 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.275 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:38.379 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:38.403 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:38.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:38.507 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:38.635 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:38.664 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:38.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:38.692 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:38 smithi125 bash[77299]: cluster 2023-10-09T06:49:37.478605+0000 mgr.y (mgr.25723) 1357 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:38.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:38 smithi125 bash[80002]: cluster 2023-10-09T06:49:37.478605+0000 mgr.y (mgr.25723) 1357 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:38.768 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:38.797 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:38.798 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:38 smithi155 bash[59495]: cluster 2023-10-09T06:49:37.478605+0000 mgr.y (mgr.25723) 1357 : cluster 0 pgmap v1485: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:38.881 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:38.900 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.918 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:38.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:39.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:39.056 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:39.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:39.159 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.180 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:39.283 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.309 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:39.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:39.440 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:39.543 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:39.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:39.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:39.701 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:39.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:39.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:39 smithi155 bash[59495]: audit 2023-10-09T06:49:38.565837+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:49:39.818 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:39 smithi155 bash[59495]: cluster 2023-10-09T06:49:39.479229+0000 mgr.y (mgr.25723) 1358 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:39.837 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:39 smithi125 bash[77299]: audit 2023-10-09T06:49:38.565837+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:49:39.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:39 smithi125 bash[77299]: cluster 2023-10-09T06:49:39.479229+0000 mgr.y (mgr.25723) 1358 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:39 smithi125 bash[80002]: audit 2023-10-09T06:49:38.565837+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:49:39.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:39 smithi125 bash[80002]: cluster 2023-10-09T06:49:39.479229+0000 mgr.y (mgr.25723) 1358 : cluster 0 pgmap v1486: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:39.941 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.965 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:39.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:40.068 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:40.194 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:40.222 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:40.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:40.325 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:40.357 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:40.461 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.486 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:40.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:40.624 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:40.728 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.753 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:40.856 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:40.884 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:40.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:40.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:41.011 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:41.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:41.140 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:41.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:41.243 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:41.278 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:41.382 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:41.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:41] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-10-09T06:49:41.407 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:41.510 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:41.534 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:41.637 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:41.672 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:41.776 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:41.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:41.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:41.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:41.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:41.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:42.052 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:42.075 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:42.178 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:42.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:42.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.350 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:42.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:42.451 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:42.475 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:42.579 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:42.604 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:42.707 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:42.735 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:42.834 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:49:42.835 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:49:42.839 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:42.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:42 smithi125 bash[77299]: cluster 2023-10-09T06:49:41.480408+0000 mgr.y (mgr.25723) 1359 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:42.852 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:42 smithi125 bash[80002]: cluster 2023-10-09T06:49:41.480408+0000 mgr.y (mgr.25723) 1359 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:42.863 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:42 smithi155 bash[59495]: cluster 2023-10-09T06:49:41.480408+0000 mgr.y (mgr.25723) 1359 : cluster 0 pgmap v1487: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:42.868 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:42.972 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:42.995 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:42.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:42.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:43.098 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:43.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.252 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:43.356 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:43.379 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:43.482 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:43.555 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:43 smithi125 bash[80002]: audit 2023-10-09T06:49:42.823208+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:49:43.572 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:49:43.608 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:43.617 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:43 smithi155 bash[59495]: audit 2023-10-09T06:49:42.823208+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:49:43.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:43.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:43.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:43.761 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:43.820 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:49:43.832 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:43 smithi125 bash[77299]: audit 2023-10-09T06:49:42.823208+0000 mon.a (mon.0) 1672 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:49:43.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:43.890 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:43.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:43.993 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:44.038 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:44.141 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:44.168 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:44.271 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:44.303 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:44.408 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:44.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:44.536 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:44.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:44.661 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:44.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: cluster 2023-10-09T06:49:43.481033+0000 mgr.y (mgr.25723) 1360 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: cluster 2023-10-09T06:49:43.547128+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: audit 2023-10-09T06:49:43.547851+0000 mon.a (mon.0) 1673 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: cluster 2023-10-09T06:49:43.547904+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e328: 8 total, 8 up, 4 in 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: cluster 2023-10-09T06:49:43.565168+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: cluster 2023-10-09T06:49:43.719277+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: audit 2023-10-09T06:49:44.202989+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: audit 2023-10-09T06:49:44.210326+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: audit 2023-10-09T06:49:44.244123+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.125:0/999938080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.674 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[80002]: audit 2023-10-09T06:49:44.244427+0000 mon.a (mon.0) 1677 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: cluster 2023-10-09T06:49:43.481033+0000 mgr.y (mgr.25723) 1360 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: cluster 2023-10-09T06:49:43.547128+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: audit 2023-10-09T06:49:43.547851+0000 mon.a (mon.0) 1673 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: cluster 2023-10-09T06:49:43.547904+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e328: 8 total, 8 up, 4 in 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: cluster 2023-10-09T06:49:43.565168+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: cluster 2023-10-09T06:49:43.719277+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: audit 2023-10-09T06:49:44.202989+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: audit 2023-10-09T06:49:44.210326+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: audit 2023-10-09T06:49:44.244123+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.125:0/999938080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:44 smithi125 bash[77299]: audit 2023-10-09T06:49:44.244427+0000 mon.a (mon.0) 1677 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:44.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:44.788 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:44.807 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: cluster 2023-10-09T06:49:43.481033+0000 mgr.y (mgr.25723) 1360 : cluster 0 pgmap v1488: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: cluster 2023-10-09T06:49:43.547128+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: audit 2023-10-09T06:49:43.547851+0000 mon.a (mon.0) 1673 : audit 1 from='client.? 172.21.15.125:0/4067965388' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: cluster 2023-10-09T06:49:43.547904+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e328: 8 total, 8 up, 4 in 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: cluster 2023-10-09T06:49:43.565168+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: cluster 2023-10-09T06:49:43.719277+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: audit 2023-10-09T06:49:44.202989+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: audit 2023-10-09T06:49:44.210326+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: audit 2023-10-09T06:49:44.244123+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.125:0/999938080' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:44 smithi155 bash[59495]: audit 2023-10-09T06:49:44.244427+0000 mon.a (mon.0) 1677 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:49:44.911 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:44.937 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:44.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:45.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:45.061 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:45.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:45.164 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:45.195 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:45.226 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is set 2023-10-09T06:49:45.299 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:45.327 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:45.431 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:45.455 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:45.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:45.558 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:45.570 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[77299]: cluster 2023-10-09T06:49:44.630779+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.208722+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[77299]: audit 2023-10-09T06:49:45.217815+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.217961+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e329: 8 total, 8 up, 4 in 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[80002]: cluster 2023-10-09T06:49:44.630779+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.208722+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[80002]: audit 2023-10-09T06:49:45.217815+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:49:45.571 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:45 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.217961+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e329: 8 total, 8 up, 4 in 2023-10-09T06:49:45.584 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:45.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:45.688 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:45.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:45.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:45.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:45.835 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:45 smithi155 bash[59495]: cluster 2023-10-09T06:49:44.630779+0000 osd.4 (osd.4) 280 : cluster 3 failed to encode map e328 with expected crc 2023-10-09T06:49:45.835 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:45 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.208722+0000 mon.a (mon.0) 1678 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:49:45.835 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:45 smithi155 bash[59495]: audit 2023-10-09T06:49:45.217815+0000 mon.a (mon.0) 1679 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:49:45.835 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:45 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.217961+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e329: 8 total, 8 up, 4 in 2023-10-09T06:49:45.855 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:45.959 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:45.989 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:45.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:46.093 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.122 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:46.225 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.253 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:46.356 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:46.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.517 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:46.621 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.215618+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.311679+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.420149+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.422662+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[77299]: cluster 2023-10-09T06:49:45.481754+0000 mgr.y (mgr.25723) 1361 : cluster 0 pgmap v1491: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.215618+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.311679+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.633 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.420149+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.422662+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:46 smithi125 bash[80002]: cluster 2023-10-09T06:49:45.481754+0000 mgr.y (mgr.25723) 1361 : cluster 0 pgmap v1491: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:46.648 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:46.751 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.781 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:46.885 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:46 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.215618+0000 osd.4 (osd.4) 281 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:46 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.311679+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:46 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.420149+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:46 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.422662+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e329 with expected crc 2023-10-09T06:49:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:46 smithi155 bash[59495]: cluster 2023-10-09T06:49:45.481754+0000 mgr.y (mgr.25723) 1361 : cluster 0 pgmap v1491: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:46.910 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:46.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:46.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:47.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:47.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:47.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:47.177 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:47.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:49:47.281 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:47.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:47.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:47.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:47.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:47.536 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:47.568 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:47.672 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:47.673 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:47 smithi125 bash[77299]: cluster 2023-10-09T06:49:47.482460+0000 mgr.y (mgr.25723) 1362 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:47.673 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:47 smithi125 bash[80002]: cluster 2023-10-09T06:49:47.482460+0000 mgr.y (mgr.25723) 1362 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:47.706 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:47.810 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:47.824 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:47 smithi155 bash[59495]: cluster 2023-10-09T06:49:47.482460+0000 mgr.y (mgr.25723) 1362 : cluster 0 pgmap v1492: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:47.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:49:47.845 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:47.848 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:49:47.868 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:49:47.892 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:49:47.918 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:49:47.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:47.957 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:49:47.975 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:47.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:47.985 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:49:48.013 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:49:48.014 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.015 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:49:48.079 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:48.210 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:48.238 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:48.341 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:48.373 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:48.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:48.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:48.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:48.587 INFO:teuthology.orchestra.run.smithi125.stderr:pool 'unique_pool_5' removed 2023-10-09T06:49:48.603 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:48.621 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:48.636 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:48.740 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[77299]: audit 2023-10-09T06:49:47.795147+0000 mon.a (mon.0) 1681 : audit 0 from='client.? 172.21.15.125:0/2017654574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[77299]: audit 2023-10-09T06:49:48.077833+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.125:0/4199968341' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[77299]: audit 2023-10-09T06:49:48.078393+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[80002]: audit 2023-10-09T06:49:47.795147+0000 mon.a (mon.0) 1681 : audit 0 from='client.? 172.21.15.125:0/2017654574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[80002]: audit 2023-10-09T06:49:48.077833+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.125:0/4199968341' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:48 smithi125 bash[80002]: audit 2023-10-09T06:49:48.078393+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:48.862 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.885 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:48.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:48 smithi155 bash[59495]: audit 2023-10-09T06:49:47.795147+0000 mon.a (mon.0) 1681 : audit 0 from='client.? 172.21.15.125:0/2017654574' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:49:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:48 smithi155 bash[59495]: audit 2023-10-09T06:49:48.077833+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.125:0/4199968341' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:48 smithi155 bash[59495]: audit 2023-10-09T06:49:48.078393+0000 mon.a (mon.0) 1682 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:49:48.988 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:49.010 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:49.114 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:49.146 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:49.165 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:49:49.250 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.284 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:49.388 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:49.415 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:49.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:49.518 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:49.547 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:49.650 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:49.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: audit 2023-10-09T06:49:48.559948+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:49:49.760 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:48.560027+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e330: 8 total, 8 up, 4 in 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: audit 2023-10-09T06:49:48.611998+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:48.632385+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:48.634878+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:48.920709+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:48.921632+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.761 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[77299]: cluster 2023-10-09T06:49:49.483059+0000 mgr.y (mgr.25723) 1363 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:49.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: audit 2023-10-09T06:49:48.559948+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:49:49.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:48.560027+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e330: 8 total, 8 up, 4 in 2023-10-09T06:49:49.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: audit 2023-10-09T06:49:48.611998+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:49.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:48.632385+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.762 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:48.634878+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:48.920709+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:48.921632+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.763 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:49 smithi125 bash[80002]: cluster 2023-10-09T06:49:49.483059+0000 mgr.y (mgr.25723) 1363 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:49.786 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:49.811 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:49.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: audit 2023-10-09T06:49:48.559948+0000 mon.a (mon.0) 1683 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:49:49.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:48.560027+0000 mon.a (mon.0) 1684 : cluster 0 osdmap e330: 8 total, 8 up, 4 in 2023-10-09T06:49:49.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: audit 2023-10-09T06:49:48.611998+0000 mon.a (mon.0) 1685 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:49:49.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:48.632385+0000 osd.4 (osd.4) 282 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:48.634878+0000 osd.4 (osd.4) 283 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:48.920709+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:48.921632+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:49.878 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:49 smithi155 bash[59495]: cluster 2023-10-09T06:49:49.483059+0000 mgr.y (mgr.25723) 1363 : cluster 0 pgmap v1494: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:49.914 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:49.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:50.047 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:50.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:50.181 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:50.209 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:50.312 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:50.339 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:50.409 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:49:50.409 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T06:49:50.410 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:49:50.410 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:49:50.410 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:49:50.411 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:49:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.424 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:49:50.425 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:49:50.443 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:50.470 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:50.490 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:49:50.490 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:49:50.539 INFO:tasks.workunit:timeout=3h 2023-10-09T06:49:50.540 INFO:tasks.workunit:cleanup=True 2023-10-09T06:49:50.540 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:49:50.573 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:50.609 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:50.611 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:49:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:50.713 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:50.746 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:50.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:50.860 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:50 smithi125 bash[77299]: cluster 2023-10-09T06:49:49.266974+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.860 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:50 smithi125 bash[77299]: cluster 2023-10-09T06:49:49.336544+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:50 smithi125 bash[80002]: cluster 2023-10-09T06:49:49.266974+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:50 smithi125 bash[80002]: cluster 2023-10-09T06:49:49.336544+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.879 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:50.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:50.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:50 smithi155 bash[59495]: cluster 2023-10-09T06:49:49.266974+0000 osd.5 (osd.5) 236 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:50 smithi155 bash[59495]: cluster 2023-10-09T06:49:49.336544+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e330 with expected crc 2023-10-09T06:49:50.983 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:51.017 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:51.121 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:51.154 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:51.259 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:51.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:51.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:49:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:49:51] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-10-09T06:49:51.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:51.422 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:51.525 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:51.558 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:51.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:51.695 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:51.765 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:49:51.778 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:51 smithi125 bash[77299]: cluster 2023-10-09T06:49:51.484050+0000 mgr.y (mgr.25723) 1364 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:51.779 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:51 smithi125 bash[80002]: cluster 2023-10-09T06:49:51.484050+0000 mgr.y (mgr.25723) 1364 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:51.799 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:51.824 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:51 smithi155 bash[59495]: cluster 2023-10-09T06:49:51.484050+0000 mgr.y (mgr.25723) 1364 : cluster 0 pgmap v1495: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:51.928 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:51.959 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:51.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:52.062 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:52.090 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:52.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:52.193 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:52.223 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:52.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:52.326 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:52.331 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:49:52.331 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:49:52.331 INFO:teuthology.orchestra.run.smithi155.stdout: "num_ops": 0 2023-10-09T06:49:52.331 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:49:52.356 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:52.460 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:52.492 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:52.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:52.596 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:52.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:52.731 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:52.759 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:52.862 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:52.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:52.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:52.997 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:53.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:53.132 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:53.158 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:53.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:53.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:53.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:53.297 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:49:53.396 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:53.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:53.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:53.560 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:53.663 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:53.693 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:53.797 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:53.827 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:53.929 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:53.931 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:53.957 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:53.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:54.060 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:54.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:54.203 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:54.237 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:54.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:54.352 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: audit 2023-10-09T06:49:53.131082+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: audit 2023-10-09T06:49:53.139178+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: cluster 2023-10-09T06:49:53.484534+0000 mgr.y (mgr.25723) 1365 : cluster 0 pgmap v1496: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:54.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: audit 2023-10-09T06:49:53.608248+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:49:54.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: audit 2023-10-09T06:49:53.610196+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:49:54.353 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:54 smithi155 bash[59495]: audit 2023-10-09T06:49:53.620223+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.372 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:54.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:54.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: audit 2023-10-09T06:49:53.131082+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: audit 2023-10-09T06:49:53.139178+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: cluster 2023-10-09T06:49:53.484534+0000 mgr.y (mgr.25723) 1365 : cluster 0 pgmap v1496: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: audit 2023-10-09T06:49:53.608248+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: audit 2023-10-09T06:49:53.610196+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[80002]: audit 2023-10-09T06:49:53.620223+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: audit 2023-10-09T06:49:53.131082+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: audit 2023-10-09T06:49:53.139178+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: cluster 2023-10-09T06:49:53.484534+0000 mgr.y (mgr.25723) 1365 : cluster 0 pgmap v1496: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: audit 2023-10-09T06:49:53.608248+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:49:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: audit 2023-10-09T06:49:53.610196+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:49:54.388 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:54 smithi125 bash[77299]: audit 2023-10-09T06:49:53.620223+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:49:54.476 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:54.504 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:54.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:54.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:54.633 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:54.737 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:54.768 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:54.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:54.872 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:54.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:54.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:54.939 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:55.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:55.045 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:55.148 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:55.173 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:55.276 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:55.308 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:55.412 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:55.444 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:55.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:55.547 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:55.677 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:55.805 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:55.838 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:55.943 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.954 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:49:55.969 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:55.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:56.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:56.098 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:56.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:56.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:56.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.367 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:56.471 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.501 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:56.605 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:56.622 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:56.700 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:49:56.706 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:49:56.707 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:49:56.708 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:49:56.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.743 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:56.846 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.855 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:56 smithi125 bash[77299]: cluster 2023-10-09T06:49:55.485531+0000 mgr.y (mgr.25723) 1366 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:56.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:56 smithi125 bash[80002]: cluster 2023-10-09T06:49:55.485531+0000 mgr.y (mgr.25723) 1366 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:56.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:56.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:56 smithi155 bash[59495]: cluster 2023-10-09T06:49:55.485531+0000 mgr.y (mgr.25723) 1366 : cluster 0 pgmap v1497: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:56.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.993 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:56.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:57.085 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:49:57.097 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:57.117 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:49:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:49:57.210 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:49:57.221 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:57.255 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:57.345 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:49:57.348 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:49:57.348 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:49:57.359 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:57.384 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:57.487 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:57.506 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:57.575 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:49:57.580 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T06:49:57.581 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:49:57.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.635 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:57 smithi125 bash[77299]: cluster 2023-10-09T06:49:57.486049+0000 mgr.y (mgr.25723) 1367 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:57.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:57 smithi125 bash[80002]: cluster 2023-10-09T06:49:57.486049+0000 mgr.y (mgr.25723) 1367 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:57.652 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T06:49:57.653 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T06:49:57.653 DEBUG:teuthology.orchestra.run.smithi125:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T06:49:57.739 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.766 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:57.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:57 smithi155 bash[59495]: cluster 2023-10-09T06:49:57.486049+0000 mgr.y (mgr.25723) 1367 : cluster 0 pgmap v1498: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:57.899 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:57.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:57.921 INFO:tasks.workunit.client.0.smithi125.stdout:Running main() from gmock_main.cc 2023-10-09T06:49:57.921 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T06:49:57.921 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment set-up. 2023-10-09T06:49:57.921 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T06:49:58.003 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.026 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:58.121 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:49:58.129 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:58.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:58.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:58.283 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:58.387 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.408 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:58.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.542 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:58.644 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:58.676 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:58.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:58.808 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:49:58.911 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:58.937 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:58.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:59.040 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:59.066 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:49:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:49:59.169 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:59.193 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:59.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:59.296 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:59.326 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:49:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:49:59.430 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:59.456 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:49:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:49:59.559 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:59.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:49:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:49:59.615 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T06:49:59.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:59.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[80002]: cluster 2023-10-09T06:49:58.604140+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[80002]: cluster 2023-10-09T06:49:58.605232+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[80002]: cluster 2023-10-09T06:49:58.605793+0000 mon.a (mon.0) 1691 : cluster 0 osdmap e331: 8 total, 8 up, 4 in 2023-10-09T06:49:59.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[80002]: audit 2023-10-09T06:49:58.606856+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:49:59.702 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.486635+0000 mgr.y (mgr.25723) 1368 : cluster 0 pgmap v1500: 105 pgs: 32 unknown, 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:59.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[77299]: cluster 2023-10-09T06:49:58.604140+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[77299]: cluster 2023-10-09T06:49:58.605232+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[77299]: cluster 2023-10-09T06:49:58.605793+0000 mon.a (mon.0) 1691 : cluster 0 osdmap e331: 8 total, 8 up, 4 in 2023-10-09T06:49:59.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[77299]: audit 2023-10-09T06:49:58.606856+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:49:59.703 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:49:59 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.486635+0000 mgr.y (mgr.25723) 1368 : cluster 0 pgmap v1500: 105 pgs: 32 unknown, 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:59.715 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:49:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:49:59.818 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:59.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:59 smithi155 bash[59495]: cluster 2023-10-09T06:49:58.604140+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.832 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:59 smithi155 bash[59495]: cluster 2023-10-09T06:49:58.605232+0000 osd.4 (osd.4) 284 : cluster 3 failed to encode map e331 with expected crc 2023-10-09T06:49:59.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:59 smithi155 bash[59495]: cluster 2023-10-09T06:49:58.605793+0000 mon.a (mon.0) 1691 : cluster 0 osdmap e331: 8 total, 8 up, 4 in 2023-10-09T06:49:59.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:59 smithi155 bash[59495]: audit 2023-10-09T06:49:58.606856+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:49:59.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:49:59 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.486635+0000 mgr.y (mgr.25723) 1368 : cluster 0 pgmap v1500: 105 pgs: 32 unknown, 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:49:59.850 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:49:59.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:49:59.953 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:59.977 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:49:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:49:59.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:00.080 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.087 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_all_features (472 ms) 2023-10-09T06:50:00.087 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T06:50:00.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:00.209 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.232 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:00.314 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.copyup (226 ms) 2023-10-09T06:50:00.314 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T06:50:00.333 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparse_copyup (19 ms) 2023-10-09T06:50:00.333 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T06:50:00.335 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.343 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_and_set_id (11 ms) 2023-10-09T06:50:00.344 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T06:50:00.359 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:00.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:00.462 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:00.484 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:00.588 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:00.609 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.add_remove_child (266 ms) 2023-10-09T06:50:00.609 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T06:50:00.615 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:00.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:00.664 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.directory_methods (54 ms) 2023-10-09T06:50:00.664 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.create 2023-10-09T06:50:00.719 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:00.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:00.796 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.create (133 ms) 2023-10-09T06:50:00.796 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T06:50:00.801 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2023-10-09T06:50:00.801 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T06:50:00.804 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_object_prefix (4 ms) 2023-10-09T06:50:00.805 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T06:50:00.807 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_create_timestamp (3 ms) 2023-10-09T06:50:00.807 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T06:50:00.810 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T06:50:00.810 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T06:50:00.812 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_modify_timestamp (3 ms) 2023-10-09T06:50:00.812 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T06:50:00.818 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_data_pool (5 ms) 2023-10-09T06:50:00.818 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T06:50:00.825 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.get_size (8 ms) 2023-10-09T06:50:00.825 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T06:50:00.832 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_size (7 ms) 2023-10-09T06:50:00.832 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T06:50:00.848 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:00.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.607289+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.607968+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: audit 2023-10-09T06:49:59.611407+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:50:00.859 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.611605+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e332: 8 total, 8 up, 4 in 2023-10-09T06:50:00.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.613042+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.613152+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:49:59.613296+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:50:00.000096+0000 mon.a (mon.0) 1695 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.860 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:00 smithi155 bash[59495]: cluster 2023-10-09T06:50:00.000176+0000 mon.a (mon.0) 1696 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.864 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.protection_status (31 ms) 2023-10-09T06:50:00.864 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T06:50:00.877 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshot_limits (13 ms) 2023-10-09T06:50:00.877 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T06:50:00.877 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:00.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:00.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.607289+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.607968+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: audit 2023-10-09T06:49:59.611407+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.611605+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e332: 8 total, 8 up, 4 in 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.613042+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.613152+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:49:59.613296+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:50:00.000096+0000 mon.a (mon.0) 1695 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[77299]: cluster 2023-10-09T06:50:00.000176+0000 mon.a (mon.0) 1696 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.607289+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.607968+0000 osd.4 (osd.4) 285 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: audit 2023-10-09T06:49:59.611407+0000 mon.a (mon.0) 1693 : audit 1 from='client.? 172.21.15.125:0/4152572583' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi125-243751-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.611605+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e332: 8 total, 8 up, 4 in 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.613042+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.613152+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:49:59.613296+0000 osd.4 (osd.4) 286 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:50:00.000096+0000 mon.a (mon.0) 1695 : cluster 3 Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:00 smithi125 bash[80002]: cluster 2023-10-09T06:50:00.000176+0000 mon.a (mon.0) 1696 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-09T06:50:00.946 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v1 (70 ms) 2023-10-09T06:50:00.947 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T06:50:00.980 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:00.996 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.parents_v2 (49 ms) 2023-10-09T06:50:00.996 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T06:50:01.006 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:01.032 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapshots (37 ms) 2023-10-09T06:50:01.032 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T06:50:01.040 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.snapid_race (8 ms) 2023-10-09T06:50:01.041 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T06:50:01.054 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.stripingv2 (13 ms) 2023-10-09T06:50:01.054 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T06:50:01.057 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-10-09T06:50:01.057 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T06:50:01.065 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_resize (9 ms) 2023-10-09T06:50:01.065 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T06:50:01.073 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2023-10-09T06:50:01.073 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T06:50:01.073 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T06:50:01.073 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T06:50:01.080 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_add (8 ms) 2023-10-09T06:50:01.080 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T06:50:01.089 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.object_map_snap_remove (9 ms) 2023-10-09T06:50:01.089 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T06:50:01.098 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.flags (8 ms) 2023-10-09T06:50:01.098 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T06:50:01.109 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:01.131 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:01.161 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.metadata (63 ms) 2023-10-09T06:50:01.161 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T06:50:01.174 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.set_features (13 ms) 2023-10-09T06:50:01.174 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T06:50:01.235 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:01.252 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror (78 ms) 2023-10-09T06:50:01.252 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T06:50:01.265 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:01.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:01.276 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image (24 ms) 2023-10-09T06:50:01.276 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T06:50:01.370 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:01.395 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:01.499 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:01.507 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:01] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-10-09T06:50:01.524 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:01.627 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:01.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:01.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:01.781 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:01.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[77299]: cluster 2023-10-09T06:50:00.625533+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T06:50:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[77299]: cluster 2023-10-09T06:50:00.626372+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e333: 8 total, 8 up, 4 in 2023-10-09T06:50:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[77299]: cluster 2023-10-09T06:50:00.628593+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[77299]: cluster 2023-10-09T06:50:01.487048+0000 mgr.y (mgr.25723) 1369 : cluster 0 pgmap v1503: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[80002]: cluster 2023-10-09T06:50:00.625533+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T06:50:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[80002]: cluster 2023-10-09T06:50:00.626372+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e333: 8 total, 8 up, 4 in 2023-10-09T06:50:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[80002]: cluster 2023-10-09T06:50:00.628593+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:01.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:01 smithi125 bash[80002]: cluster 2023-10-09T06:50:01.487048+0000 mgr.y (mgr.25723) 1369 : cluster 0 pgmap v1503: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:01.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:01 smithi155 bash[59495]: cluster 2023-10-09T06:50:00.625533+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e333 with expected crc 2023-10-09T06:50:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:01 smithi155 bash[59495]: cluster 2023-10-09T06:50:00.626372+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e333: 8 total, 8 up, 4 in 2023-10-09T06:50:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:01 smithi155 bash[59495]: cluster 2023-10-09T06:50:00.628593+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e332 with expected crc 2023-10-09T06:50:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:01 smithi155 bash[59495]: cluster 2023-10-09T06:50:01.487048+0000 mgr.y (mgr.25723) 1369 : cluster 0 pgmap v1503: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:01.909 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:01.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:02.012 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:02.038 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:02.141 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:02.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:02.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:02.300 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:02.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:02.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:02.531 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.558 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:02.661 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:02.687 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:02.790 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.791 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:50:02.791 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [], 2023-10-09T06:50:02.791 INFO:teuthology.orchestra.run.smithi155.stdout: "complaint_time": 30, 2023-10-09T06:50:02.792 INFO:teuthology.orchestra.run.smithi155.stdout: "num_blocked_ops": 0 2023-10-09T06:50:02.792 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:50:02.816 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:02.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.943 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:02.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:03.047 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:03.071 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:03.175 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.200 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:03.304 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:03.329 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:03.433 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:03.458 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:03.562 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:03.587 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:03.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:03.816 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:03.840 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:03.946 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.970 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:50:03.978 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:03.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:04.082 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:04.106 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:04.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:04.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:04.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:04.368 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:04.472 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:04.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:04.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:04 smithi155 bash[59495]: cluster 2023-10-09T06:50:03.487593+0000 mgr.y (mgr.25723) 1370 : cluster 0 pgmap v1504: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:04.612 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:04 smithi155 bash[59495]: audit 2023-10-09T06:50:03.612525+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:04.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:04.734 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:04 smithi125 bash[77299]: cluster 2023-10-09T06:50:03.487593+0000 mgr.y (mgr.25723) 1370 : cluster 0 pgmap v1504: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:04.746 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:04 smithi125 bash[77299]: audit 2023-10-09T06:50:03.612525+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:04.746 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:04 smithi125 bash[80002]: cluster 2023-10-09T06:50:03.487593+0000 mgr.y (mgr.25723) 1370 : cluster 0 pgmap v1504: 105 pgs: 16 creating+activating, 89 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:04.747 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:04 smithi125 bash[80002]: audit 2023-10-09T06:50:03.612525+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:04.760 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:04.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.887 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:04.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:04.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.017 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:05.120 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:05.152 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:05.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:05.255 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:05.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:05.340 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_status (4065 ms) 2023-10-09T06:50:05.340 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T06:50:05.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:05.409 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:05.490 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_image_map (149 ms) 2023-10-09T06:50:05.490 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T06:50:05.505 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_instances (16 ms) 2023-10-09T06:50:05.505 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T06:50:05.512 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.526 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.mirror_snapshot (20 ms) 2023-10-09T06:50:05.526 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T06:50:05.530 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_list (4 ms) 2023-10-09T06:50:05.530 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T06:50:05.533 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-09T06:50:05.533 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T06:50:05.538 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2023-10-09T06:50:05.539 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T06:50:05.539 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:05.550 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_rename (12 ms) 2023-10-09T06:50:05.550 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T06:50:05.556 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-09T06:50:05.556 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T06:50:05.562 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (7 ms) 2023-10-09T06:50:05.563 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T06:50:05.566 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-10-09T06:50:05.566 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T06:50:05.572 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_remove (6 ms) 2023-10-09T06:50:05.572 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T06:50:05.576 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-09T06:50:05.577 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T06:50:05.584 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_image_clean (8 ms) 2023-10-09T06:50:05.584 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T06:50:05.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T06:50:05.588 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T06:50:05.593 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_remove (5 ms) 2023-10-09T06:50:05.593 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T06:50:05.596 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-10-09T06:50:05.597 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T06:50:05.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-10-09T06:50:05.599 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T06:50:05.602 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-09T06:50:05.602 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T06:50:05.607 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (5 ms) 2023-10-09T06:50:05.607 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T06:50:05.613 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (6 ms) 2023-10-09T06:50:05.613 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T06:50:05.617 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T06:50:05.617 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T06:50:05.622 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-09T06:50:05.623 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T06:50:05.642 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:05.653 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (30 ms) 2023-10-09T06:50:05.653 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T06:50:05.671 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:05.774 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:05.797 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:05.901 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.930 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:05.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:05.984 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (332 ms) 2023-10-09T06:50:05.985 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T06:50:05.990 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_remove (6 ms) 2023-10-09T06:50:05.990 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T06:50:05.994 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-10-09T06:50:05.994 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T06:50:06.006 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.trash_methods (11 ms) 2023-10-09T06:50:06.006 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T06:50:06.018 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.op_features (13 ms) 2023-10-09T06:50:06.018 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T06:50:06.035 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:06.061 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent (43 ms) 2023-10-09T06:50:06.061 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T06:50:06.063 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:06.074 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_parent_ns (13 ms) 2023-10-09T06:50:06.075 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T06:50:06.094 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.clone_child (19 ms) 2023-10-09T06:50:06.094 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T06:50:06.106 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.namespace_methods (12 ms) 2023-10-09T06:50:06.106 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T06:50:06.122 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration (17 ms) 2023-10-09T06:50:06.122 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T06:50:06.136 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-10-09T06:50:06.136 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T06:50:06.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:06.198 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:06.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:06.331 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:06.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:06.434 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:06.454 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:06.557 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:06.561 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:50:06.582 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:06 smithi155 bash[59495]: cluster 2023-10-09T06:50:05.488362+0000 mgr.y (mgr.25723) 1371 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 1.18k op/s 2023-10-09T06:50:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:06 smithi155 bash[59495]: audit 2023-10-09T06:50:06.354838+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.125:0/228859959' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:06 smithi155 bash[59495]: audit 2023-10-09T06:50:06.355357+0000 mon.a (mon.0) 1699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.686 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:06.712 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[77299]: cluster 2023-10-09T06:50:05.488362+0000 mgr.y (mgr.25723) 1371 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 1.18k op/s 2023-10-09T06:50:06.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[77299]: audit 2023-10-09T06:50:06.354838+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.125:0/228859959' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.713 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[77299]: audit 2023-10-09T06:50:06.355357+0000 mon.a (mon.0) 1699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.713 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[80002]: cluster 2023-10-09T06:50:05.488362+0000 mgr.y (mgr.25723) 1371 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 1.18k op/s 2023-10-09T06:50:06.713 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[80002]: audit 2023-10-09T06:50:06.354838+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.125:0/228859959' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.713 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:06 smithi125 bash[80002]: audit 2023-10-09T06:50:06.355357+0000 mon.a (mon.0) 1699 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:50:06.718 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:06.824 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:06.854 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:06.958 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:06.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:06.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:07.089 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.120 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:07.224 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.251 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:07.355 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:07.381 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:07.483 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:07.511 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:07.574 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1438 ms) 2023-10-09T06:50:07.574 INFO:tasks.workunit.client.0.smithi125.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T06:50:07.614 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.634 INFO:tasks.workunit.client.0.smithi125.stdout:[ OK ] TestClsRbd.sparsify (60 ms) 2023-10-09T06:50:07.642 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:07.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:07.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:07.884 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[77299]: audit 2023-10-09T06:50:06.551318+0000 mon.a (mon.0) 1700 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[77299]: cluster 2023-10-09T06:50:06.551440+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e334: 8 total, 8 up, 4 in 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[77299]: cluster 2023-10-09T06:50:06.553581+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[77299]: cluster 2023-10-09T06:50:06.556558+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[77299]: cluster 2023-10-09T06:50:07.488826+0000 mgr.y (mgr.25723) 1372 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.6 MiB/s rd, 924 KiB/s wr, 1.03k op/s 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[80002]: audit 2023-10-09T06:50:06.551318+0000 mon.a (mon.0) 1700 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[80002]: cluster 2023-10-09T06:50:06.551440+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e334: 8 total, 8 up, 4 in 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[80002]: cluster 2023-10-09T06:50:06.553581+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[80002]: cluster 2023-10-09T06:50:06.556558+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:07 smithi125 bash[80002]: cluster 2023-10-09T06:50:07.488826+0000 mgr.y (mgr.25723) 1372 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.6 MiB/s rd, 924 KiB/s wr, 1.03k op/s 2023-10-09T06:50:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:07 smithi155 bash[59495]: audit 2023-10-09T06:50:06.551318+0000 mon.a (mon.0) 1700 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:50:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:07 smithi155 bash[59495]: cluster 2023-10-09T06:50:06.551440+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e334: 8 total, 8 up, 4 in 2023-10-09T06:50:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:07 smithi155 bash[59495]: cluster 2023-10-09T06:50:06.553581+0000 osd.4 (osd.4) 287 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:07 smithi155 bash[59495]: cluster 2023-10-09T06:50:06.556558+0000 osd.4 (osd.4) 288 : cluster 3 failed to encode map e334 with expected crc 2023-10-09T06:50:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:07 smithi155 bash[59495]: cluster 2023-10-09T06:50:07.488826+0000 mgr.y (mgr.25723) 1372 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1.6 MiB/s rd, 924 KiB/s wr, 1.03k op/s 2023-10-09T06:50:07.908 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:07.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:07.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:08.011 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:08.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:08.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:08.171 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:08.274 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:08.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:08.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:08.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:08.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:08.563 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:08.577 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] 70 tests from TestClsRbd (8019 ms total) 2023-10-09T06:50:08.577 INFO:tasks.workunit.client.0.smithi125.stdout: 2023-10-09T06:50:08.577 INFO:tasks.workunit.client.0.smithi125.stdout:[----------] Global test environment tear-down 2023-10-09T06:50:08.577 INFO:tasks.workunit.client.0.smithi125.stdout:[==========] 70 tests from 1 test suite ran. (10657 ms total) 2023-10-09T06:50:08.577 INFO:tasks.workunit.client.0.smithi125.stdout:[ PASSED ] 70 tests. 2023-10-09T06:50:08.579 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:50:08.580 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:50:08.633 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T06:50:08.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:50:08.667 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[77299]: cluster 2023-10-09T06:50:07.569467+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[77299]: cluster 2023-10-09T06:50:07.571256+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e335: 8 total, 8 up, 4 in 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[77299]: cluster 2023-10-09T06:50:07.574096+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[80002]: cluster 2023-10-09T06:50:07.569467+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[80002]: cluster 2023-10-09T06:50:07.571256+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e335: 8 total, 8 up, 4 in 2023-10-09T06:50:08.687 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:08 smithi125 bash[80002]: cluster 2023-10-09T06:50:07.574096+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.701 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:08.753 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:50:08.806 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:08.831 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:08.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:08 smithi155 bash[59495]: cluster 2023-10-09T06:50:07.569467+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:08 smithi155 bash[59495]: cluster 2023-10-09T06:50:07.571256+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e335: 8 total, 8 up, 4 in 2023-10-09T06:50:08.833 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:08 smithi155 bash[59495]: cluster 2023-10-09T06:50:07.574096+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:08.896 DEBUG:teuthology.parallel:result is None 2023-10-09T06:50:08.897 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:08.910 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:08.911 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:08.935 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:08.959 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:08.959 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:50:08.959 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T06:50:08.959 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:50:08.959 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:50:08.959 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:50:08.960 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:08.972 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:08.978 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:50:08.978 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:09.025 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:09.025 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:50:09.073 INFO:tasks.workunit:timeout=3h 2023-10-09T06:50:09.073 INFO:tasks.workunit:cleanup=True 2023-10-09T06:50:09.074 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:50:09.075 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:09.105 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:09.126 INFO:tasks.workunit.client.0.smithi125.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:50:09.209 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:09.230 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:09.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:09.332 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:09.356 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:09.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:09.459 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:09.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:09.590 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.618 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:09.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:09.744 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:09.848 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[77299]: cluster 2023-10-09T06:50:08.076602+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.854 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[77299]: cluster 2023-10-09T06:50:08.341057+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[77299]: cluster 2023-10-09T06:50:08.572847+0000 mon.a (mon.0) 1703 : cluster 0 osdmap e336: 8 total, 8 up, 4 in 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[77299]: cluster 2023-10-09T06:50:09.489326+0000 mgr.y (mgr.25723) 1373 : cluster 0 pgmap v1510: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[80002]: cluster 2023-10-09T06:50:08.076602+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[80002]: cluster 2023-10-09T06:50:08.341057+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[80002]: cluster 2023-10-09T06:50:08.572847+0000 mon.a (mon.0) 1703 : cluster 0 osdmap e336: 8 total, 8 up, 4 in 2023-10-09T06:50:09.855 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:09 smithi125 bash[80002]: cluster 2023-10-09T06:50:09.489326+0000 mgr.y (mgr.25723) 1373 : cluster 0 pgmap v1510: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:09.865 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:09.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:09 smithi155 bash[59495]: cluster 2023-10-09T06:50:08.076602+0000 osd.4 (osd.4) 289 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:09 smithi155 bash[59495]: cluster 2023-10-09T06:50:08.341057+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e335 with expected crc 2023-10-09T06:50:09.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:09 smithi155 bash[59495]: cluster 2023-10-09T06:50:08.572847+0000 mon.a (mon.0) 1703 : cluster 0 osdmap e336: 8 total, 8 up, 4 in 2023-10-09T06:50:09.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:09 smithi155 bash[59495]: cluster 2023-10-09T06:50:09.489326+0000 mgr.y (mgr.25723) 1373 : cluster 0 pgmap v1510: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:09.968 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.988 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:09.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:10.092 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:10.119 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:10.125 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:50:10.125 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:50:10.222 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:10.248 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:10.351 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.383 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:10.407 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:50:10.429 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:50:10.460 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:50:10.486 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:50:10.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:10.516 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:10.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:10.526 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:50:10.556 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:50:10.580 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:50:10.606 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.608 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:50:10.620 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:10 smithi155 bash[59495]: cluster 2023-10-09T06:50:09.288693+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T06:50:10.632 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:10 smithi155 bash[59495]: audit 2023-10-09T06:50:10.377406+0000 mon.a (mon.0) 1704 : audit 0 from='client.? 172.21.15.125:0/1672849363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:10.648 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:10 smithi125 bash[80002]: cluster 2023-10-09T06:50:09.288693+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T06:50:10.648 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:10 smithi125 bash[80002]: audit 2023-10-09T06:50:10.377406+0000 mon.a (mon.0) 1704 : audit 0 from='client.? 172.21.15.125:0/1672849363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:10.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:10 smithi125 bash[77299]: cluster 2023-10-09T06:50:09.288693+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e336 with expected crc 2023-10-09T06:50:10.648 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:10 smithi125 bash[77299]: audit 2023-10-09T06:50:10.377406+0000 mon.a (mon.0) 1704 : audit 0 from='client.? 172.21.15.125:0/1672849363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:10.652 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:10.755 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.785 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:10.889 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:10.915 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:10.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:11.018 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:11.051 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:11.156 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:11.183 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:11.286 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:11.320 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:11.425 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:11.453 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:11.557 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:11.585 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:11] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-10-09T06:50:11.586 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:11.600 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:50:11.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:11.712 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:11.816 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:11.841 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:11.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:11 smithi125 bash[80002]: audit 2023-10-09T06:50:10.847864+0000 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:50:11.843 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:11 smithi125 bash[80002]: cluster 2023-10-09T06:50:11.490359+0000 mgr.y (mgr.25723) 1374 : cluster 0 pgmap v1511: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:11.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:11 smithi125 bash[77299]: audit 2023-10-09T06:50:10.847864+0000 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:50:11.843 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:11 smithi125 bash[77299]: cluster 2023-10-09T06:50:11.490359+0000 mgr.y (mgr.25723) 1374 : cluster 0 pgmap v1511: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:11 smithi155 bash[59495]: audit 2023-10-09T06:50:10.847864+0000 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:50:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:11 smithi155 bash[59495]: cluster 2023-10-09T06:50:11.490359+0000 mgr.y (mgr.25723) 1374 : cluster 0 pgmap v1511: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:11.944 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:11.970 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:11.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:12.073 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:12.097 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:12.201 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:12.227 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:12.331 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:12.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:12.455 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:12.481 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:12.584 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:12.607 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[80002]: cluster 2023-10-09T06:50:11.581251+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[80002]: cluster 2023-10-09T06:50:11.591802+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[80002]: audit 2023-10-09T06:50:11.593705+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[80002]: cluster 2023-10-09T06:50:11.593769+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e337: 8 total, 8 up, 4 in 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[77299]: cluster 2023-10-09T06:50:11.581251+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:50:12.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[77299]: cluster 2023-10-09T06:50:11.591802+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[77299]: audit 2023-10-09T06:50:11.593705+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:50:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:12 smithi125 bash[77299]: cluster 2023-10-09T06:50:11.593769+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e337: 8 total, 8 up, 4 in 2023-10-09T06:50:12.710 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:12.739 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:12.842 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:12.850 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:12 smithi155 bash[59495]: cluster 2023-10-09T06:50:11.581251+0000 mon.a (mon.0) 1706 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:50:12.850 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:12 smithi155 bash[59495]: cluster 2023-10-09T06:50:11.591802+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:12.850 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:12 smithi155 bash[59495]: audit 2023-10-09T06:50:11.593705+0000 mon.a (mon.0) 1707 : audit 1 from='client.? 172.21.15.125:0/1767978212' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:50:12.851 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:12 smithi155 bash[59495]: cluster 2023-10-09T06:50:11.593769+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e337: 8 total, 8 up, 4 in 2023-10-09T06:50:12.866 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:12.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:12.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:13.000 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:13.104 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:13.127 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:13.231 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.260 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:13.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.390 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:13.490 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:50:13.490 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:50:13.494 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.523 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:13.626 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:13 smithi155 bash[59495]: cluster 2023-10-09T06:50:12.441685+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:13 smithi155 bash[59495]: cluster 2023-10-09T06:50:12.590945+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:13 smithi155 bash[59495]: cluster 2023-10-09T06:50:12.590994+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:13 smithi155 bash[59495]: cluster 2023-10-09T06:50:13.490894+0000 mgr.y (mgr.25723) 1375 : cluster 0 pgmap v1513: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:13.653 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:13.757 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:13.779 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:13.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:13.882 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[80002]: cluster 2023-10-09T06:50:12.441685+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[80002]: cluster 2023-10-09T06:50:12.590945+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[80002]: cluster 2023-10-09T06:50:12.590994+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[80002]: cluster 2023-10-09T06:50:13.490894+0000 mgr.y (mgr.25723) 1375 : cluster 0 pgmap v1513: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[77299]: cluster 2023-10-09T06:50:12.441685+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[77299]: cluster 2023-10-09T06:50:12.590945+0000 osd.4 (osd.4) 290 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[77299]: cluster 2023-10-09T06:50:12.590994+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e337 with expected crc 2023-10-09T06:50:13.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:13 smithi125 bash[77299]: cluster 2023-10-09T06:50:13.490894+0000 mgr.y (mgr.25723) 1375 : cluster 0 pgmap v1513: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:13.913 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:13.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:13.952 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:50:14.016 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:14.046 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:14.150 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:14.172 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:14.275 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:14.307 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:14.410 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:14.435 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:14.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:50:14.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:14.696 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.724 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:14.828 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.836 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:50:14.844 INFO:tasks.workunit.client.0.smithi125.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:50:14.847 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:50:14.847 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:50:14.863 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:14.918 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:50:14.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:50:14.966 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.984 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:50:14.996 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:14.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:15.045 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:50:15.046 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:50:15.046 INFO:tasks.workunit.client.0.smithi125.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:50:15.099 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.102 INFO:tasks.workunit.client.0.smithi125.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:50:15.105 DEBUG:teuthology.orchestra.run.smithi125:> set -ex 2023-10-09T06:50:15.105 DEBUG:teuthology.orchestra.run.smithi125:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:50:15.129 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:15.135 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T06:50:15.136 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T06:50:15.136 DEBUG:teuthology.orchestra.run.smithi125:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T06:50:15.193 INFO:tasks.workunit.client.0.smithi125.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T06:50:15.242 INFO:tasks.workunit.client.0.smithi125.stderr:+ TMPDIR=/tmp/rbd_import_export_245068 2023-10-09T06:50:15.242 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_245068 2023-10-09T06:50:15.242 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:15.243 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir /tmp/rbd_import_export_245068 2023-10-09T06:50:15.243 INFO:tasks.workunit.client.0.smithi125.stderr:+ trap rm -rf /tmp/rbd_import_export_245068 INT TERM EXIT 2023-10-09T06:50:15.243 INFO:tasks.workunit.client.0.smithi125.stderr:+ mkdir foo.245068 2023-10-09T06:50:15.244 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import foo.245068 foo.dir 2023-10-09T06:50:15.277 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:15.289 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: cannot import a directory 2023-10-09T06:50:15.289 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 0% complete...failed. 2023-10-09T06:50:15.289 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: import failed: (21) Is a directory 2023-10-09T06:50:15.293 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:15.293 INFO:tasks.workunit.client.0.smithi125.stderr:+ rmdir foo.245068 2023-10-09T06:50:15.294 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_245068/img bs=1k count=1 seek=10 2023-10-09T06:50:15.295 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:50:15.295 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:50:15.295 INFO:tasks.workunit.client.0.smithi125.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000133408 s, 7.7 MB/s 2023-10-09T06:50:15.295 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_245068/img bs=1k count=10 seek=100 2023-10-09T06:50:15.296 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T06:50:15.297 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T06:50:15.297 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000173243 s, 59.1 MB/s 2023-10-09T06:50:15.297 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_245068/img bs=1k count=100 seek=1000 2023-10-09T06:50:15.298 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records in 2023-10-09T06:50:15.298 INFO:tasks.workunit.client.0.smithi125.stderr:70+1 records out 2023-10-09T06:50:15.298 INFO:tasks.workunit.client.0.smithi125.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000355606 s, 203 MB/s 2023-10-09T06:50:15.298 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_245068/img bs=1k seek=10000 2023-10-09T06:50:15.299 INFO:tasks.workunit.client.0.smithi125.stderr:138+1 records in 2023-10-09T06:50:15.299 INFO:tasks.workunit.client.0.smithi125.stderr:138+1 records out 2023-10-09T06:50:15.300 INFO:tasks.workunit.client.0.smithi125.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000792609 s, 179 MB/s 2023-10-09T06:50:15.300 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_245068/img bs=1k seek=100000 2023-10-09T06:50:15.308 INFO:tasks.workunit.client.0.smithi125.stderr:74+1 records in 2023-10-09T06:50:15.308 INFO:tasks.workunit.client.0.smithi125.stderr:74+1 records out 2023-10-09T06:50:15.308 INFO:tasks.workunit.client.0.smithi125.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.00766651 s, 9.9 MB/s 2023-10-09T06:50:15.309 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_245068/img bs=1k seek=1000000 2023-10-09T06:50:15.317 INFO:tasks.workunit.client.0.smithi125.stderr:194+1 records in 2023-10-09T06:50:15.317 INFO:tasks.workunit.client.0.smithi125.stderr:194+1 records out 2023-10-09T06:50:15.317 INFO:tasks.workunit.client.0.smithi125.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.00786239 s, 25.3 MB/s 2023-10-09T06:50:15.318 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:50:15.376 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:50:15.376 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:50:15.381 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.382 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:15.382 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:15.410 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:15.515 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.547 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:15.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:15.677 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:15.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:15.780 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:15.812 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:15.916 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:15.948 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:15.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:16.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:16.087 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:16.107 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T06:50:16.109 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:16.113 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_245068/img2 2023-10-09T06:50:16.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.216 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:16.320 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:16.342 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:16.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:16.444 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:16.446 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:16.480 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:16.494 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:16.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T06:50:16.583 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:16.612 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:16.715 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.728 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:16 smithi125 bash[77299]: cluster 2023-10-09T06:50:15.491926+0000 mgr.y (mgr.25723) 1376 : cluster 0 pgmap v1514: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:16.729 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:16 smithi125 bash[80002]: cluster 2023-10-09T06:50:15.491926+0000 mgr.y (mgr.25723) 1376 : cluster 0 pgmap v1514: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:16.741 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:16.845 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.867 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:16.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:16.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:16 smithi155 bash[59495]: cluster 2023-10-09T06:50:15.491926+0000 mgr.y (mgr.25723) 1376 : cluster 0 pgmap v1514: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:16.971 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:17.005 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:17.108 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:17.136 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:17.240 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.269 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:17.373 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.405 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:17.443 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:17.447 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:50:17.510 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:17.564 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:17.667 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.684 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:50:17.690 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img2 2023-10-09T06:50:17.694 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:17.798 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.829 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:17.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:17.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:17 smithi125 bash[80002]: cluster 2023-10-09T06:50:17.492503+0000 mgr.y (mgr.25723) 1377 : cluster 0 pgmap v1515: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:17 smithi125 bash[77299]: cluster 2023-10-09T06:50:17.492503+0000 mgr.y (mgr.25723) 1377 : cluster 0 pgmap v1515: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:17.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:17 smithi155 bash[59495]: cluster 2023-10-09T06:50:17.492503+0000 mgr.y (mgr.25723) 1377 : cluster 0 pgmap v1515: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:17.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:17.962 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:17.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:18.065 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.099 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:18.203 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:18.234 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:18.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.369 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:18.473 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:18.474 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:18.490 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img3 2023-10-09T06:50:18.506 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:18.610 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:18.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:18.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:18.740 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:18.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:18.883 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:18 smithi125 bash[77299]: audit 2023-10-09T06:50:18.612841+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:18.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:18 smithi125 bash[80002]: audit 2023-10-09T06:50:18.612841+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:18.901 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:18.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:19.004 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.035 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:19.051 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img2 /tmp/rbd_import_export_245068/img3 2023-10-09T06:50:19.138 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:18 smithi155 bash[59495]: audit 2023-10-09T06:50:18.612841+0000 mon.a (mon.0) 1709 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:19.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:19.258 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:50:19.270 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.298 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:19.401 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.427 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:19.488 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:19.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.557 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:19.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:19.682 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:19.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:19.786 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:19.810 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:19.914 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:19.941 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:19.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:19.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:20.044 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:20.077 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:20.181 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:20.213 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:20.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:20.317 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:20.331 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:20 smithi155 bash[59495]: cluster 2023-10-09T06:50:19.493077+0000 mgr.y (mgr.25723) 1378 : cluster 0 pgmap v1516: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:20.351 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:20.454 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:20.483 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:20.587 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:20.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:20 smithi125 bash[77299]: cluster 2023-10-09T06:50:19.493077+0000 mgr.y (mgr.25723) 1378 : cluster 0 pgmap v1516: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:20.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:20 smithi125 bash[80002]: cluster 2023-10-09T06:50:19.493077+0000 mgr.y (mgr.25723) 1378 : cluster 0 pgmap v1516: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:20.616 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:20.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:20.747 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:20.851 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:20.886 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:20.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:20.989 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:21.023 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:21.127 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.163 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:21.267 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:21.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:21.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:21.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:21.531 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:21.542 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:21] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-10-09T06:50:21.561 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:21.665 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.697 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:21.800 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:21.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:21.933 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.967 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:21.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:22.072 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:22.104 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:22.207 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:22.232 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:22.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:22.336 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:22.363 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:22.466 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:22.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:22.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:22.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:22.631 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:22.734 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:22.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:22.869 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:22.877 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:22 smithi155 bash[59495]: cluster 2023-10-09T06:50:21.494167+0000 mgr.y (mgr.25723) 1379 : cluster 0 pgmap v1517: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 121 KiB/s rd, 52 KiB/s wr, 30 op/s 2023-10-09T06:50:22.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:22 smithi125 bash[77299]: cluster 2023-10-09T06:50:21.494167+0000 mgr.y (mgr.25723) 1379 : cluster 0 pgmap v1517: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 121 KiB/s rd, 52 KiB/s wr, 30 op/s 2023-10-09T06:50:22.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:22 smithi125 bash[80002]: cluster 2023-10-09T06:50:21.494167+0000 mgr.y (mgr.25723) 1379 : cluster 0 pgmap v1517: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 121 KiB/s rd, 52 KiB/s wr, 30 op/s 2023-10-09T06:50:22.893 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:22.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:22.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.025 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:23.128 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.158 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:23.262 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:23.296 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:23.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:23.365 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:50:23.365 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:50:23.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:23.433 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:23.537 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:23.570 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:23.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:23.699 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:23.803 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:23.815 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:23 smithi125 bash[80002]: cluster 2023-10-09T06:50:23.494881+0000 mgr.y (mgr.25723) 1380 : cluster 0 pgmap v1518: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 102 KiB/s rd, 44 KiB/s wr, 25 op/s 2023-10-09T06:50:23.815 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:23 smithi125 bash[77299]: cluster 2023-10-09T06:50:23.494881+0000 mgr.y (mgr.25723) 1380 : cluster 0 pgmap v1518: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 102 KiB/s rd, 44 KiB/s wr, 25 op/s 2023-10-09T06:50:23.830 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:23 smithi155 bash[59495]: cluster 2023-10-09T06:50:23.494881+0000 mgr.y (mgr.25723) 1380 : cluster 0 pgmap v1518: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 102 KiB/s rd, 44 KiB/s wr, 25 op/s 2023-10-09T06:50:23.935 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.970 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:23.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:24.074 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:24.103 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:24.206 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:24.236 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:24.293 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:50:24.340 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:24.374 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:24.419 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format - testimg 2023-10-09T06:50:24.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:24.498 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:24.601 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:24.632 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:24.735 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:24.766 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:24.870 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:24.897 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:24.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:24.950 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:24.950 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:24.955 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg /tmp/rbd_import_export_245068/img2 2023-10-09T06:50:25.001 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.037 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:25.140 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.165 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:25.268 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.288 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:25.292 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg - 2023-10-09T06:50:25.295 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:25.399 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:25.426 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:25.530 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:25.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:25.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:25.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:25 smithi125 bash[77299]: cluster 2023-10-09T06:50:25.495470+0000 mgr.y (mgr.25723) 1381 : cluster 0 pgmap v1519: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:25.670 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:25 smithi125 bash[80002]: cluster 2023-10-09T06:50:25.495470+0000 mgr.y (mgr.25723) 1381 : cluster 0 pgmap v1519: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:25.682 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:25.785 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:25 smithi155 bash[59495]: cluster 2023-10-09T06:50:25.495470+0000 mgr.y (mgr.25723) 1381 : cluster 0 pgmap v1519: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:25.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:25.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:25.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:26.059 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:26.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:26.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:26.218 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:26.222 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:50:26.223 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:26.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:26.351 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:26.370 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:50:26.373 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img2 2023-10-09T06:50:26.455 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:26.487 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:26.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:26.591 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:26.619 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:26.722 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:26.749 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:26.853 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:26.886 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:26.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:26.990 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:27.016 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:27.119 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:27.124 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img3 2023-10-09T06:50:27.148 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:27.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:27.280 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:27.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:27.385 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.423 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:27.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:27.523 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:27.555 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:27.638 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img2 /tmp/rbd_import_export_245068/img3 2023-10-09T06:50:27.659 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:27.685 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:27.788 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:27.819 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:27.922 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:27.953 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:27.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:28.056 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:28.089 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:28.193 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:28.224 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:28.327 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:28.358 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:28.386 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd help export 2023-10-09T06:50:28.387 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -q export-format 2023-10-09T06:50:28.462 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:28.487 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:28.590 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:28.619 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:28.723 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:28.751 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:28.799 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_245068/img bs=1k count=10 seek=100 2023-10-09T06:50:28.800 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records in 2023-10-09T06:50:28.800 INFO:tasks.workunit.client.0.smithi125.stderr:10+0 records out 2023-10-09T06:50:28.800 INFO:tasks.workunit.client.0.smithi125.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000106756 s, 95.9 MB/s 2023-10-09T06:50:28.800 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:28.840 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:28 smithi125 bash[77299]: cluster 2023-10-09T06:50:27.495903+0000 mgr.y (mgr.25723) 1382 : cluster 0 pgmap v1520: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:28.841 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:28 smithi125 bash[80002]: cluster 2023-10-09T06:50:27.495903+0000 mgr.y (mgr.25723) 1382 : cluster 0 pgmap v1520: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:28.854 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:28.873 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:28.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:28.878 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:28.879 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:28.881 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap 2023-10-09T06:50:28.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:28 smithi155 bash[59495]: cluster 2023-10-09T06:50:27.495903+0000 mgr.y (mgr.25723) 1382 : cluster 0 pgmap v1520: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:28.976 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:29.008 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:29.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:29.078 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:50:29.112 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:29.136 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:29.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:29.239 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:29.261 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:29.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:29.367 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:29.388 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:29.491 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:29.525 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:29.578 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:50:29.588 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T06:50:29.628 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:29.650 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:29.653 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta list testimg 2023-10-09T06:50:29.699 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-09T06:50:29.699 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:29.699 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:50:29.699 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 2023-10-09T06:50:29.699 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:29.750 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:29.753 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:29.754 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_245068/img_v2 testimg_import 2023-10-09T06:50:29.784 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:29.888 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:29.911 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:29.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:30.013 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:30.044 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:30.149 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:30.176 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:30.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:30.291 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:30 smithi155 bash[59495]: cluster 2023-10-09T06:50:29.496567+0000 mgr.y (mgr.25723) 1383 : cluster 0 pgmap v1521: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:30.292 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:30 smithi155 bash[59495]: cluster 2023-10-09T06:50:29.555692+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e338: 8 total, 8 up, 4 in 2023-10-09T06:50:30.306 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:30.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:30.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:30 smithi125 bash[77299]: cluster 2023-10-09T06:50:29.496567+0000 mgr.y (mgr.25723) 1383 : cluster 0 pgmap v1521: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:30.422 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:30 smithi125 bash[77299]: cluster 2023-10-09T06:50:29.555692+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e338: 8 total, 8 up, 4 in 2023-10-09T06:50:30.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:30 smithi125 bash[80002]: cluster 2023-10-09T06:50:29.496567+0000 mgr.y (mgr.25723) 1383 : cluster 0 pgmap v1521: 73 pgs: 73 active+clean; 9.2 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 130 KiB/s rd, 71 KiB/s wr, 49 op/s 2023-10-09T06:50:30.422 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:30 smithi125 bash[80002]: cluster 2023-10-09T06:50:29.555692+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e338: 8 total, 8 up, 4 in 2023-10-09T06:50:30.436 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:30.539 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:30.575 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:30.587 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:50:30.594 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:50:30.634 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: id: 8d99e2b0e8a1 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.8d99e2b0e8a1 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.635 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.636 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.638 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap 2023-10-09T06:50:30.678 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout:rbd image 'testimg_import': 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: size 110 KiB in 1 objects 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: order 22 (4 MiB objects) 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: snapshot_count: 1 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: id: 8d99e2b0e8a1 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: block_name_prefix: rbd_data.8d99e2b0e8a1 2023-10-09T06:50:30.684 INFO:tasks.workunit.client.0.smithi125.stdout: format: 2 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: op_features: 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: flags: 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: create_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: access_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: modify_timestamp: Mon Oct 9 06:50:29 2023 2023-10-09T06:50:30.685 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T06:50:30.690 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd image-meta list testimg_import 2023-10-09T06:50:30.704 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:30.740 INFO:tasks.workunit.client.0.smithi125.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ There is 1 metadatum on this image: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:Key Value 2023-10-09T06:50:30.741 INFO:tasks.workunit.client.0.smithi125.stderr:key1 value1 ] 2023-10-09T06:50:30.742 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import /tmp/rbd_import_export_245068/img_import 2023-10-09T06:50:30.781 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:50:30.785 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img_import 2023-10-09T06:50:30.785 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/img /tmp/rbd_import_export_245068/img_import 2023-10-09T06:50:30.786 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/img --format %b 2023-10-09T06:50:30.787 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=24 2023-10-09T06:50:30.787 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/img_import --format %b 2023-10-09T06:50:30.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T06:50:30.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T06:50:30.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=200 2023-10-09T06:50:30.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T06:50:30.789 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_245068/img_snap 2023-10-09T06:50:30.807 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:30.829 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:30.835 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:50:30.839 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_245068/img_snap_import 2023-10-09T06:50:30.879 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:50:30.885 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/img_snap /tmp/rbd_import_export_245068/img_snap_import 2023-10-09T06:50:30.885 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/img_snap /tmp/rbd_import_export_245068/img_snap_import 2023-10-09T06:50:30.886 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/img_snap --format %b 2023-10-09T06:50:30.888 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=224 2023-10-09T06:50:30.888 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/img_snap_import --format %b 2023-10-09T06:50:30.889 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=224 2023-10-09T06:50:30.889 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:30.890 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:30.890 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:50:30.890 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:30.890 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_import 2023-10-09T06:50:30.891 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_snap 2023-10-09T06:50:30.891 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_snap_import 2023-10-09T06:50:30.892 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T06:50:30.933 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:30.958 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:30.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:31.061 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:31.091 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:31.195 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:31.226 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:31.329 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:31.363 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:31.387 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:31] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-10-09T06:50:31.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[77299]: cluster 2023-10-09T06:50:29.562373+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T06:50:31.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[77299]: cluster 2023-10-09T06:50:30.558299+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[77299]: cluster 2023-10-09T06:50:30.559612+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e339: 8 total, 8 up, 4 in 2023-10-09T06:50:31.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[77299]: cluster 2023-10-09T06:50:30.560998+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[80002]: cluster 2023-10-09T06:50:29.562373+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T06:50:31.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[80002]: cluster 2023-10-09T06:50:30.558299+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[80002]: cluster 2023-10-09T06:50:30.559612+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e339: 8 total, 8 up, 4 in 2023-10-09T06:50:31.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:31 smithi125 bash[80002]: cluster 2023-10-09T06:50:30.560998+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.466 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:31.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:31 smithi155 bash[59495]: cluster 2023-10-09T06:50:29.562373+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e338 with expected crc 2023-10-09T06:50:31.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:31 smithi155 bash[59495]: cluster 2023-10-09T06:50:30.558299+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:31 smithi155 bash[59495]: cluster 2023-10-09T06:50:30.559612+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e339: 8 total, 8 up, 4 in 2023-10-09T06:50:31.480 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:31 smithi155 bash[59495]: cluster 2023-10-09T06:50:30.560998+0000 osd.4 (osd.4) 291 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:31.499 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:31.575 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T06:50:31.590 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:50:31.604 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:31.640 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:31.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:31.726 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:31.730 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap rm testimg@snap 2023-10-09T06:50:31.744 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:31.768 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:31.871 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:31.899 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:31.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:32.002 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.030 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:32.133 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:32.159 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:32.262 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:32.294 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:32.397 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:32.428 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:32.532 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:32.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[77299]: cluster 2023-10-09T06:50:30.562629+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:32.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[77299]: cluster 2023-10-09T06:50:31.497269+0000 mgr.y (mgr.25723) 1384 : cluster 0 pgmap v1524: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 101 op/s 2023-10-09T06:50:32.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[77299]: cluster 2023-10-09T06:50:31.560368+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[77299]: cluster 2023-10-09T06:50:31.563850+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e340: 8 total, 8 up, 4 in 2023-10-09T06:50:32.542 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[77299]: cluster 2023-10-09T06:50:31.574910+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[80002]: cluster 2023-10-09T06:50:30.562629+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:32.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[80002]: cluster 2023-10-09T06:50:31.497269+0000 mgr.y (mgr.25723) 1384 : cluster 0 pgmap v1524: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 101 op/s 2023-10-09T06:50:32.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[80002]: cluster 2023-10-09T06:50:31.560368+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[80002]: cluster 2023-10-09T06:50:31.563850+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e340: 8 total, 8 up, 4 in 2023-10-09T06:50:32.543 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:32 smithi125 bash[80002]: cluster 2023-10-09T06:50:31.574910+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:32.580 INFO:tasks.workunit.client.0.smithi125.stderr: Removing snap: 100% complete...done. 2023-10-09T06:50:32.592 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm testimg 2023-10-09T06:50:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:32 smithi155 bash[59495]: cluster 2023-10-09T06:50:30.562629+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e339 with expected crc 2023-10-09T06:50:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:32 smithi155 bash[59495]: cluster 2023-10-09T06:50:31.497269+0000 mgr.y (mgr.25723) 1384 : cluster 0 pgmap v1524: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 200 KiB/s rd, 71 KiB/s wr, 101 op/s 2023-10-09T06:50:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:32 smithi155 bash[59495]: cluster 2023-10-09T06:50:31.560368+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:32 smithi155 bash[59495]: cluster 2023-10-09T06:50:31.563850+0000 mon.a (mon.0) 1712 : cluster 0 osdmap e340: 8 total, 8 up, 4 in 2023-10-09T06:50:32.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:32 smithi155 bash[59495]: cluster 2023-10-09T06:50:31.574910+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:32.660 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:32.687 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:32.722 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:32.726 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --order 20 /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:32.791 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.809 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:32.810 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:32.813 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:32.820 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:32.890 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:32.894 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_245068/img_v2 testimg_import 2023-10-09T06:50:32.924 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.955 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:32.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:32.975 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:50:32.980 INFO:tasks.workunit.client.0.smithi125.stderr:+ + rbdgrep order 2023-10-09T06:50:32.980 INFO:tasks.workunit.client.0.smithi125.stderr: info testimg_import 2023-10-09T06:50:32.980 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $2} 2023-10-09T06:50:32.980 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 20 2023-10-09T06:50:33.060 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:33.081 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:33.184 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:33.205 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:33.254 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:50:33.255 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:50:33.307 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.336 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:33.429 INFO:tasks.workunit.client.0.smithi125.stdout:20 2023-10-09T06:50:33.429 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:33.430 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:50:33.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:33 smithi155 bash[59495]: cluster 2023-10-09T06:50:31.566326+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:33.447 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:33 smithi155 bash[59495]: cluster 2023-10-09T06:50:32.562913+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e341: 8 total, 8 up, 4 in 2023-10-09T06:50:33.466 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:33 smithi125 bash[77299]: cluster 2023-10-09T06:50:31.566326+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:33.467 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:33 smithi125 bash[77299]: cluster 2023-10-09T06:50:32.562913+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e341: 8 total, 8 up, 4 in 2023-10-09T06:50:33.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:33 smithi125 bash[80002]: cluster 2023-10-09T06:50:31.566326+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e340 with expected crc 2023-10-09T06:50:33.468 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:33 smithi125 bash[80002]: cluster 2023-10-09T06:50:32.562913+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e341: 8 total, 8 up, 4 in 2023-10-09T06:50:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:33.570 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:33.571 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:33.576 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:50:33.595 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:33.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.712 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:33.716 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:33.720 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:33.779 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:33.782 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg 2023-10-09T06:50:33.783 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep features 2023-10-09T06:50:33.822 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:33.832 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_BEFORE= features: layering 2023-10-09T06:50:33.832 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: 2023-10-09T06:50:33.832 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:33.847 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:33.950 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:33.982 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:33.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:33.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:34.054 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:34.058 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_245068/img_v2 testimg_import 2023-10-09T06:50:34.085 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:34.107 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:34.135 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:50:34.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:50:34.141 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep features 2023-10-09T06:50:34.186 INFO:tasks.workunit.client.0.smithi125.stderr:+ FEATURES_AFTER= features: layering 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ features: layering 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: != features: layering 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr: op_features: ] 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:34.187 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:50:34.211 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:34.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:34 smithi155 bash[59495]: cluster 2023-10-09T06:50:32.568938+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T06:50:34.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:34 smithi155 bash[59495]: cluster 2023-10-09T06:50:33.497882+0000 mgr.y (mgr.25723) 1385 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 311 KiB/s rd, 60 KiB/s wr, 130 op/s 2023-10-09T06:50:34.223 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:34 smithi155 bash[59495]: audit 2023-10-09T06:50:33.613074+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:34.239 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:34.291 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:34.296 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:50:34.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[77299]: cluster 2023-10-09T06:50:32.568938+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[77299]: cluster 2023-10-09T06:50:33.497882+0000 mgr.y (mgr.25723) 1385 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 311 KiB/s rd, 60 KiB/s wr, 130 op/s 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[77299]: audit 2023-10-09T06:50:33.613074+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[80002]: cluster 2023-10-09T06:50:32.568938+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e341 with expected crc 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[80002]: cluster 2023-10-09T06:50:33.497882+0000 mgr.y (mgr.25723) 1385 : cluster 0 pgmap v1527: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 311 KiB/s rd, 60 KiB/s wr, 130 op/s 2023-10-09T06:50:34.353 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:34 smithi125 bash[80002]: audit 2023-10-09T06:50:33.613074+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:34.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:50:34.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:34.404 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:34.408 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:34.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.493 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:34.494 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:34.497 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:34.598 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.625 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:34.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.735 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:34.740 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_245068/img_v2 testimg_import 2023-10-09T06:50:34.748 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:34.836 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:50:34.840 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:50:34.840 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep stripe unit 2023-10-09T06:50:34.840 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (4 KiB|4096) 2023-10-09T06:50:34.852 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:34.883 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:34.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:34.887 INFO:tasks.workunit.client.0.smithi125.stdout: stripe unit: 4 KiB 2023-10-09T06:50:34.888 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import 2023-10-09T06:50:34.888 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep stripe count 2023-10-09T06:50:34.888 INFO:tasks.workunit.client.0.smithi125.stderr:+ + awk {print $3} 2023-10-09T06:50:34.888 INFO:tasks.workunit.client.0.smithi125.stderr:grep 1000 2023-10-09T06:50:34.935 INFO:tasks.workunit.client.0.smithi125.stdout:1000 2023-10-09T06:50:34.935 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/img_v2 2023-10-09T06:50:34.936 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:50:34.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:35.008 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:35.073 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:50:35.078 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:50:35.111 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:35.143 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:35.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:35.222 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:50:35.226 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_245068/img testimg 2023-10-09T06:50:35.247 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:35.279 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:35.300 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:35.303 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap1 2023-10-09T06:50:35.382 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:35.412 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:35.438 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:50:35.449 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap create testimg@snap2 2023-10-09T06:50:35.516 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:35.548 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:35.652 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:35.679 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:35.783 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:35.817 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:35.919 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:35.949 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:35.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:35.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:36.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:36.086 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:36.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:36.189 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:36.222 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:36.326 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:36.353 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:36.437 INFO:tasks.workunit.client.0.smithi125.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:50:36.446 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap protect testimg@snap2 2023-10-09T06:50:36.456 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:36.468 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:36 smithi155 bash[59495]: cluster 2023-10-09T06:50:35.415761+0000 mon.a (mon.0) 1715 : cluster 0 osdmap e342: 8 total, 8 up, 4 in 2023-10-09T06:50:36.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:36 smithi155 bash[59495]: cluster 2023-10-09T06:50:35.418938+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T06:50:36.469 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:36 smithi155 bash[59495]: cluster 2023-10-09T06:50:35.498516+0000 mgr.y (mgr.25723) 1386 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 382 op/s 2023-10-09T06:50:36.485 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:36.493 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_245068/snap_protect 2023-10-09T06:50:36.541 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:36.545 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_245068/snap_protect testimg_import 2023-10-09T06:50:36.588 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:36.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[80002]: cluster 2023-10-09T06:50:35.415761+0000 mon.a (mon.0) 1715 : cluster 0 osdmap e342: 8 total, 8 up, 4 in 2023-10-09T06:50:36.595 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[80002]: cluster 2023-10-09T06:50:35.418938+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T06:50:36.596 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[80002]: cluster 2023-10-09T06:50:35.498516+0000 mgr.y (mgr.25723) 1386 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 382 op/s 2023-10-09T06:50:36.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[77299]: cluster 2023-10-09T06:50:35.415761+0000 mon.a (mon.0) 1715 : cluster 0 osdmap e342: 8 total, 8 up, 4 in 2023-10-09T06:50:36.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[77299]: cluster 2023-10-09T06:50:35.418938+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e342 with expected crc 2023-10-09T06:50:36.596 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:36 smithi125 bash[77299]: cluster 2023-10-09T06:50:35.498516+0000 mgr.y (mgr.25723) 1386 : cluster 0 pgmap v1529: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 382 op/s 2023-10-09T06:50:36.608 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:36.711 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:36.737 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:36.840 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:36.866 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:36.969 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:36.992 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:36.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:37.095 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:37.129 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:37.233 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:37.263 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:37.366 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:37.402 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:37.506 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:37.532 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:37.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:37.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:37.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:37 smithi155 bash[59495]: cluster 2023-10-09T06:50:36.418100+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:37 smithi155 bash[59495]: cluster 2023-10-09T06:50:36.418710+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:37 smithi155 bash[59495]: cluster 2023-10-09T06:50:36.420224+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e343: 8 total, 8 up, 4 in 2023-10-09T06:50:37.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:37 smithi155 bash[59495]: cluster 2023-10-09T06:50:36.422832+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.664 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:37.767 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:37.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[77299]: cluster 2023-10-09T06:50:36.418100+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[77299]: cluster 2023-10-09T06:50:36.418710+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[77299]: cluster 2023-10-09T06:50:36.420224+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e343: 8 total, 8 up, 4 in 2023-10-09T06:50:37.779 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[77299]: cluster 2023-10-09T06:50:36.422832+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[80002]: cluster 2023-10-09T06:50:36.418100+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[80002]: cluster 2023-10-09T06:50:36.418710+0000 osd.4 (osd.4) 292 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[80002]: cluster 2023-10-09T06:50:36.420224+0000 mon.a (mon.0) 1716 : cluster 0 osdmap e343: 8 total, 8 up, 4 in 2023-10-09T06:50:37.780 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:37 smithi125 bash[80002]: cluster 2023-10-09T06:50:36.422832+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e343 with expected crc 2023-10-09T06:50:37.794 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:37.896 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:37.924 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:37.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:38.027 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:38.059 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:38.162 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:38.188 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:38.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:38.292 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:38.328 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:38.432 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:38.460 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:38.474 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:50:38.478 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info testimg_import@snap1 2023-10-09T06:50:38.478 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep protected: False 2023-10-09T06:50:38.522 INFO:tasks.workunit.client.0.smithi125.stdout: protected: False 2023-10-09T06:50:38.523 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grep protected: True 2023-10-09T06:50:38.523 INFO:tasks.workunit.client.0.smithi125.stderr:rbd info testimg_import@snap2 2023-10-09T06:50:38.563 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:38.567 INFO:tasks.workunit.client.0.smithi125.stdout: protected: True 2023-10-09T06:50:38.567 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/snap_protect 2023-10-09T06:50:38.569 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T06:50:38.586 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:38.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T06:50:38.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:38 smithi155 bash[59495]: cluster 2023-10-09T06:50:37.429589+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e344: 8 total, 8 up, 4 in 2023-10-09T06:50:38.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:38 smithi155 bash[59495]: cluster 2023-10-09T06:50:37.432645+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T06:50:38.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:38 smithi155 bash[59495]: cluster 2023-10-09T06:50:37.499213+0000 mgr.y (mgr.25723) 1387 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 383 op/s 2023-10-09T06:50:38.677 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg 2023-10-09T06:50:38.689 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:38.720 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:50:38.721 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:50:38.722 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:38.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:38.826 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:38.859 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[77299]: cluster 2023-10-09T06:50:37.429589+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e344: 8 total, 8 up, 4 in 2023-10-09T06:50:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[77299]: cluster 2023-10-09T06:50:37.432645+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T06:50:38.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[77299]: cluster 2023-10-09T06:50:37.499213+0000 mgr.y (mgr.25723) 1387 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 383 op/s 2023-10-09T06:50:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[80002]: cluster 2023-10-09T06:50:37.429589+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e344: 8 total, 8 up, 4 in 2023-10-09T06:50:38.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[80002]: cluster 2023-10-09T06:50:37.432645+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e344 with expected crc 2023-10-09T06:50:38.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:38 smithi125 bash[80002]: cluster 2023-10-09T06:50:37.499213+0000 mgr.y (mgr.25723) 1387 : cluster 0 pgmap v1532: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 293 KiB/s rd, 32 KiB/s wr, 383 op/s 2023-10-09T06:50:38.963 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:38.989 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:38.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:39.093 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:39.125 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:39.152 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:50:39.229 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:39.258 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:39.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:39.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:39.386 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:39.489 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:39.517 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:39.620 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[77299]: cluster 2023-10-09T06:50:38.438032+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e345: 8 total, 8 up, 4 in 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[77299]: cluster 2023-10-09T06:50:38.440505+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[77299]: cluster 2023-10-09T06:50:38.450155+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[77299]: cluster 2023-10-09T06:50:39.171198+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e346: 8 total, 8 up, 4 in 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[77299]: cluster 2023-10-09T06:50:39.174390+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T06:50:39.631 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[80002]: cluster 2023-10-09T06:50:38.438032+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e345: 8 total, 8 up, 4 in 2023-10-09T06:50:39.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[80002]: cluster 2023-10-09T06:50:38.440505+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[80002]: cluster 2023-10-09T06:50:38.450155+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[80002]: cluster 2023-10-09T06:50:39.171198+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e346: 8 total, 8 up, 4 in 2023-10-09T06:50:39.632 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:39 smithi125 bash[80002]: cluster 2023-10-09T06:50:39.174390+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T06:50:39.643 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:39.746 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:39.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:39 smithi155 bash[59495]: cluster 2023-10-09T06:50:38.438032+0000 mon.a (mon.0) 1718 : cluster 0 osdmap e345: 8 total, 8 up, 4 in 2023-10-09T06:50:39.757 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:39 smithi155 bash[59495]: cluster 2023-10-09T06:50:38.440505+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:39 smithi155 bash[59495]: cluster 2023-10-09T06:50:38.450155+0000 osd.4 (osd.4) 293 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:39.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:39 smithi155 bash[59495]: cluster 2023-10-09T06:50:39.171198+0000 mon.a (mon.0) 1719 : cluster 0 osdmap e346: 8 total, 8 up, 4 in 2023-10-09T06:50:39.758 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:39 smithi155 bash[59495]: cluster 2023-10-09T06:50:39.174390+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e346 with expected crc 2023-10-09T06:50:39.771 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:39.875 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:39.907 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:39.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:40.008 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:40.034 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:40.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:40.137 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.164 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:40.203 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:50:40.215 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd snap purge testimg_import 2023-10-09T06:50:40.268 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:40.292 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:40.395 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:40.420 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:40.525 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:40.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:40 smithi155 bash[59495]: audit 2023-10-09T06:50:38.702469+0000 mgr.y (mgr.25723) 1388 : audit 0 from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:40.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:40 smithi155 bash[59495]: cluster 2023-10-09T06:50:39.171042+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:40.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:40 smithi155 bash[59495]: cluster 2023-10-09T06:50:39.499844+0000 mgr.y (mgr.25723) 1389 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:40.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:40 smithi155 bash[59495]: cluster 2023-10-09T06:50:40.182359+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e347: 8 total, 8 up, 4 in 2023-10-09T06:50:40.537 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:40 smithi155 bash[59495]: cluster 2023-10-09T06:50:40.192332+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:40.560 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:40.664 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[77299]: audit 2023-10-09T06:50:38.702469+0000 mgr.y (mgr.25723) 1388 : audit 0 from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:40.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[77299]: cluster 2023-10-09T06:50:39.171042+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:40.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[77299]: cluster 2023-10-09T06:50:39.499844+0000 mgr.y (mgr.25723) 1389 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:40.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[77299]: cluster 2023-10-09T06:50:40.182359+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e347: 8 total, 8 up, 4 in 2023-10-09T06:50:40.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[77299]: cluster 2023-10-09T06:50:40.192332+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:40.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[80002]: audit 2023-10-09T06:50:38.702469+0000 mgr.y (mgr.25723) 1388 : audit 0 from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:40.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[80002]: cluster 2023-10-09T06:50:39.171042+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e345 with expected crc 2023-10-09T06:50:40.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[80002]: cluster 2023-10-09T06:50:39.499844+0000 mgr.y (mgr.25723) 1389 : cluster 0 pgmap v1535: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:50:40.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[80002]: cluster 2023-10-09T06:50:40.182359+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e347: 8 total, 8 up, 4 in 2023-10-09T06:50:40.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:40 smithi125 bash[80002]: cluster 2023-10-09T06:50:40.192332+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:40.689 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:40.792 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.820 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:40.843 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:50:40.864 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:50:40.887 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:50:40.912 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:50:40.923 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.934 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:50:40.948 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:40.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:40.963 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:50:40.982 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1533,"stamp":"2023-10-09T06:50:38.434373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4800533,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":8853,"num_read_kb":16912,"num_write":1490,"num_write_kb":16406,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":356,"num_bytes_recovered":7768541,"num_keys_recovered":20,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1611,"ondisk_log_size":1611,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3656112,"kb_used_data":26720,"kb_used_omap":8,"kb_used_meta":3629303,"kb_avail":371324496,"statfs":{"total":383980142592,"available":380236283904,"internally_reserved":0,"allocated":27361280,"data_stored":24226514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":3716406528},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-215044,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1660,"num_read_kb":1447,"num_write":232,"num_write_kb":157,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.936850"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573231+0000","last_change":"2023-10-09T06:45:05.136158+0000","last_active":"2023-10-09T06:50:31.573231+0000","last_peered":"2023-10-09T06:50:31.573231+0000","last_clean":"2023-10-09T06:50:31.573231+0000","last_became_active":"2023-10-09T06:45:05.135763+0000","last_became_peered":"2023-10-09T06:45:05.135763+0000","last_unstale":"2023-10-09T06:50:31.573231+0000","last_undegraded":"2023-10-09T06:50:31.573231+0000","last_fullsized":"2023-10-09T06:50:31.573231+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":321,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573941+0000","last_change":"2023-10-09T06:45:33.529264+0000","last_active":"2023-10-09T06:50:31.573941+0000","last_peered":"2023-10-09T06:50:31.573941+0000","last_clean":"2023-10-09T06:50:31.573941+0000","last_became_active":"2023-10-09T06:45:33.528935+0000","last_became_peered":"2023-10-09T06:45:33.528935+0000","last_unstale":"2023-10-09T06:50:31.573941+0000","last_undegraded":"2023-10-09T06:50:31.573941+0000","last_fullsized":"2023-10-09T06:50:31.573941+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572613+0000","last_change":"2023-10-09T06:49:20.588657+0000","last_active":"2023-10-09T06:50:31.572613+0000","last_peered":"2023-10-09T06:50:31.572613+0000","last_clean":"2023-10-09T06:50:31.572613+0000","last_became_active":"2023-10-09T06:49:20.588048+0000","last_became_peered":"2023-10-09T06:49:20.588048+0000","last_unstale":"2023-10-09T06:50:31.572613+0000","last_undegraded":"2023-10-09T06:50:31.572613+0000","last_fullsized":"2023-10-09T06:50:31.572613+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":207,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570750+0000","last_change":"2023-10-09T06:45:33.542604+0000","last_active":"2023-10-09T06:50:31.570750+0000","last_peered":"2023-10-09T06:50:31.570750+0000","last_clean":"2023-10-09T06:50:31.570750+0000","last_became_active":"2023-10-09T06:45:33.541873+0000","last_became_peered":"2023-10-09T06:45:33.541873+0000","last_unstale":"2023-10-09T06:50:31.570750+0000","last_undegraded":"2023-10-09T06:50:31.570750+0000","last_fullsized":"2023-10-09T06:50:31.570750+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":369,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.568752+0000","last_change":"2023-10-09T06:48:57.562856+0000","last_active":"2023-10-09T06:50:31.568752+0000","last_peered":"2023-10-09T06:50:31.568752+0000","last_clean":"2023-10-09T06:50:31.568752+0000","last_became_active":"2023-10-09T06:48:57.562492+0000","last_became_peered":"2023-10-09T06:48:57.562492+0000","last_unstale":"2023-10-09T06:50:31.568752+0000","last_undegraded":"2023-10-09T06:50:31.568752+0000","last_fullsized":"2023-10-09T06:50:31.568752+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.578560+0000","last_change":"2023-10-09T06:45:33.532461+0000","last_active":"2023-10-09T06:50:31.578560+0000","last_peered":"2023-10-09T06:50:31.578560+0000","last_clean":"2023-10-09T06:50:31.578560+0000","last_became_active":"2023-10-09T06:45:33.532017+0000","last_became_peered":"2023-10-09T06:45:33.532017+0000","last_unstale":"2023-10-09T06:50:31.578560+0000","last_undegraded":"2023-10-09T06:50:31.578560+0000","last_fullsized":"2023-10-09T06:50:31.578560+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":305,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.578636+0000","last_change":"2023-10-09T06:45:33.522075+0000","last_active":"2023-10-09T06:50:31.578636+0000","last_peered":"2023-10-09T06:50:31.578636+0000","last_clean":"2023-10-09T06:50:31.578636+0000","last_became_active":"2023-10-09T06:45:33.521697+0000","last_became_peered":"2023-10-09T06:45:33.521697+0000","last_unstale":"2023-10-09T06:50:31.578636+0000","last_undegraded":"2023-10-09T06:50:31.578636+0000","last_fullsized":"2023-10-09T06:50:31.578636+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":352,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577524+0000","last_change":"2023-10-09T06:45:33.532746+0000","last_active":"2023-10-09T06:50:31.577524+0000","last_peered":"2023-10-09T06:50:31.577524+0000","last_clean":"2023-10-09T06:50:31.577524+0000","last_became_active":"2023-10-09T06:45:33.532241+0000","last_became_peered":"2023-10-09T06:45:33.532241+0000","last_unstale":"2023-10-09T06:50:31.577524+0000","last_undegraded":"2023-10-09T06:50:31.577524+0000","last_fullsized":"2023-10-09T06:50:31.577524+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570661+0000","last_change":"2023-10-09T06:49:20.583506+0000","last_active":"2023-10-09T06:50:31.570661+0000","last_peered":"2023-10-09T06:50:31.570661+0000","last_clean":"2023-10-09T06:50:31.570661+0000","last_became_active":"2023-10-09T06:49:20.583097+0000","last_became_peered":"2023-10-09T06:49:20.583097+0000","last_unstale":"2023-10-09T06:50:31.570661+0000","last_undegraded":"2023-10-09T06:50:31.570661+0000","last_fullsized":"2023-10-09T06:50:31.570661+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":338,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569138+0000","last_change":"2023-10-09T06:45:33.529016+0000","last_active":"2023-10-09T06:50:31.569138+0000","last_peered":"2023-10-09T06:50:31.569138+0000","last_clean":"2023-10-09T06:50:31.569138+0000","last_became_active":"2023-10-09T06:45:33.528659+0000","last_became_peered":"2023-10-09T06:45:33.528659+0000","last_unstale":"2023-10-09T06:50:31.569138+0000","last_undegraded":"2023-10-09T06:50:31.569138+0000","last_fullsized":"2023-10-09T06:50:31.569138+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":254,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570458+0000","last_change":"2023-10-09T06:45:33.810240+0000","last_active":"2023-10-09T06:50:31.570458+0000","last_peered":"2023-10-09T06:50:31.570458+0000","last_clean":"2023-10-09T06:50:31.570458+0000","last_became_active":"2023-10-09T06:45:33.538067+0000","last_became_peered":"2023-10-09T06:45:33.538067+0000","last_unstale":"2023-10-09T06:50:31.570458+0000","last_undegraded":"2023-10-09T06:50:31.570458+0000","last_fullsized":"2023-10-09T06:50:31.570458+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":96,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577958+0000","last_change":"2023-10-09T06:45:05.139216+0000","last_active":"2023-10-09T06:50:31.577958+0000","last_peered":"2023-10-09T06:50:31.577958+0000","last_clean":"2023-10-09T06:50:31.577958+0000","last_became_active":"2023-10-09T06:45:05.138786+0000","last_became_peered":"2023-10-09T06:45:05.138786+0000","last_unstale":"2023-10-09T06:50:31.577958+0000","last_undegraded":"2023-10-09T06:50:31.577958+0000","last_fullsized":"2023-10-09T06:50:31.577958+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":427,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573628+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:50:31.573628+0000","last_peered":"2023-10-09T06:50:31.573628+0000","last_clean":"2023-10-09T06:50:31.573628+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:50:31.573628+0000","last_undegraded":"2023-10-09T06:50:31.573628+0000","last_fullsized":"2023-10-09T06:50:31.573628+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":359,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573860+0000","last_change":"2023-10-09T06:45:05.135916+0000","last_active":"2023-10-09T06:50:31.573860+0000","last_peered":"2023-10-09T06:50:31.573860+0000","last_clean":"2023-10-09T06:50:31.573860+0000","last_became_active":"2023-10-09T06:45:05.135486+0000","last_became_peered":"2023-10-09T06:45:05.135486+0000","last_unstale":"2023-10-09T06:50:31.573860+0000","last_undegraded":"2023-10-09T06:50:31.573860+0000","last_fullsized":"2023-10-09T06:50:31.573860+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":254,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569357+0000","last_change":"2023-10-09T06:45:05.231838+0000","last_active":"2023-10-09T06:50:31.569357+0000","last_peered":"2023-10-09T06:50:31.569357+0000","last_clean":"2023-10-09T06:50:31.569357+0000","last_became_active":"2023-10-09T06:45:05.139440+0000","last_became_peered":"2023-10-09T06:45:05.139440+0000","last_unstale":"2023-10-09T06:50:31.569357+0000","last_undegraded":"2023-10-09T06:50:31.569357+0000","last_fullsized":"2023-10-09T06:50:31.569357+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":353,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569438+0000","last_change":"2023-10-09T06:45:33.533944+0000","last_active":"2023-10-09T06:50:31.569438+0000","last_peered":"2023-10-09T06:50:31.569438+0000","last_clean":"2023-10-09T06:50:31.569438+0000","last_became_active":"2023-10-09T06:45:33.533570+0000","last_became_peered":"2023-10-09T06:45:33.533570+0000","last_unstale":"2023-10-09T06:50:31.569438+0000","last_undegraded":"2023-10-09T06:50:31.569438+0000","last_fullsized":"2023-10-09T06:50:31.569438+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":198,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569557+0000","last_change":"2023-10-09T06:45:33.542558+0000","last_active":"2023-10-09T06:50:31.569557+0000","last_peered":"2023-10-09T06:50:31.569557+0000","last_clean":"2023-10-09T06:50:31.569557+0000","last_became_active":"2023-10-09T06:45:33.542133+0000","last_became_peered":"2023-10-09T06:45:33.542133+0000","last_unstale":"2023-10-09T06:50:31.569557+0000","last_undegraded":"2023-10-09T06:50:31.569557+0000","last_fullsized":"2023-10-09T06:50:31.569557+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":238,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570340+0000","last_change":"2023-10-09T06:45:33.533983+0000","last_active":"2023-10-09T06:50:31.570340+0000","last_peered":"2023-10-09T06:50:31.570340+0000","last_clean":"2023-10-09T06:50:31.570340+0000","last_became_active":"2023-10-09T06:45:33.533641+0000","last_became_peered":"2023-10-09T06:45:33.533641+0000","last_unstale":"2023-10-09T06:50:31.570340+0000","last_undegraded":"2023-10-09T06:50:31.570340+0000","last_fullsized":"2023-10-09T06:50:31.570340+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":287,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.568927+0000","last_change":"2023-10-09T06:45:05.141190+0000","last_active":"2023-10-09T06:50:31.568927+0000","last_peered":"2023-10-09T06:50:31.568927+0000","last_clean":"2023-10-09T06:50:31.568927+0000","last_became_active":"2023-10-09T06:45:05.140792+0000","last_became_peered":"2023-10-09T06:45:05.140792+0000","last_unstale":"2023-10-09T06:50:31.568927+0000","last_undegraded":"2023-10-09T06:50:31.568927+0000","last_fullsized":"2023-10-09T06:50:31.568927+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":138,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570526+0000","last_change":"2023-10-09T06:45:33.542855+0000","last_active":"2023-10-09T06:50:31.570526+0000","last_peered":"2023-10-09T06:50:31.570526+0000","last_clean":"2023-10-09T06:50:31.570526+0000","last_became_active":"2023-10-09T06:45:33.542281+0000","last_became_peered":"2023-10-09T06:45:33.542281+0000","last_unstale":"2023-10-09T06:50:31.570526+0000","last_undegraded":"2023-10-09T06:50:31.570526+0000","last_fullsized":"2023-10-09T06:50:31.570526+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":136,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577682+0000","last_change":"2023-10-09T06:48:58.571870+0000","last_active":"2023-10-09T06:50:31.577682+0000","last_peered":"2023-10-09T06:50:31.577682+0000","last_clean":"2023-10-09T06:50:31.577682+0000","last_became_active":"2023-10-09T06:48:58.571360+0000","last_became_peered":"2023-10-09T06:48:58.571360+0000","last_unstale":"2023-10-09T06:50:31.577682+0000","last_undegraded":"2023-10-09T06:50:31.577682+0000","last_fullsized":"2023-10-09T06:50:31.577682+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":375,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569856+0000","last_change":"2023-10-09T06:48:57.566980+0000","last_active":"2023-10-09T06:50:31.569856+0000","last_peered":"2023-10-09T06:50:31.569856+0000","last_clean":"2023-10-09T06:50:31.569856+0000","last_became_active":"2023-10-09T06:48:57.566645+0000","last_became_peered":"2023-10-09T06:48:57.566645+0000","last_unstale":"2023-10-09T06:50:31.569856+0000","last_undegraded":"2023-10-09T06:50:31.569856+0000","last_fullsized":"2023-10-09T06:50:31.569856+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":237,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569815+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:50:31.569815+0000","last_peered":"2023-10-09T06:50:31.569815+0000","last_clean":"2023-10-09T06:50:31.569815+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:50:31.569815+0000","last_undegraded":"2023-10-09T06:50:31.569815+0000","last_fullsized":"2023-10-09T06:50:31.569815+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":284,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569126+0000","last_change":"2023-10-09T06:42:32.485962+0000","last_active":"2023-10-09T06:50:31.569126+0000","last_peered":"2023-10-09T06:50:31.569126+0000","last_clean":"2023-10-09T06:50:31.569126+0000","last_became_active":"2023-10-09T06:42:32.485115+0000","last_became_peered":"2023-10-09T06:42:32.485115+0000","last_unstale":"2023-10-09T06:50:31.569126+0000","last_undegraded":"2023-10-09T06:50:31.569126+0000","last_fullsized":"2023-10-09T06:50:31.569126+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":276,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570361+0000","last_change":"2023-10-09T06:49:20.633058+0000","last_active":"2023-10-09T06:50:31.570361+0000","last_peered":"2023-10-09T06:50:31.570361+0000","last_clean":"2023-10-09T06:50:31.570361+0000","last_became_active":"2023-10-09T06:49:20.583223+0000","last_became_peered":"2023-10-09T06:49:20.583223+0000","last_unstale":"2023-10-09T06:50:31.570361+0000","last_undegraded":"2023-10-09T06:50:31.570361+0000","last_fullsized":"2023-10-09T06:50:31.570361+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":347,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573781+0000","last_change":"2023-10-09T06:45:33.535003+0000","last_active":"2023-10-09T06:50:31.573781+0000","last_peered":"2023-10-09T06:50:31.573781+0000","last_clean":"2023-10-09T06:50:31.573781+0000","last_became_active":"2023-10-09T06:45:33.533458+0000","last_became_peered":"2023-10-09T06:45:33.533458+0000","last_unstale":"2023-10-09T06:50:31.573781+0000","last_undegraded":"2023-10-09T06:50:31.573781+0000","last_fullsized":"2023-10-09T06:50:31.573781+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":315,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569473+0000","last_change":"2023-10-09T06:45:33.895855+0000","last_active":"2023-10-09T06:50:31.569473+0000","last_peered":"2023-10-09T06:50:31.569473+0000","last_clean":"2023-10-09T06:50:31.569473+0000","last_became_active":"2023-10-09T06:45:33.538145+0000","last_became_peered":"2023-10-09T06:45:33.538145+0000","last_unstale":"2023-10-09T06:50:31.569473+0000","last_undegraded":"2023-10-09T06:50:31.569473+0000","last_fullsized":"2023-10-09T06:50:31.569473+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577591+0000","last_change":"2023-10-09T06:48:57.570316+0000","last_active":"2023-10-09T06:50:31.577591+0000","last_peered":"2023-10-09T06:50:31.577591+0000","last_clean":"2023-10-09T06:50:31.577591+0000","last_became_active":"2023-10-09T06:48:57.568574+0000","last_became_peered":"2023-10-09T06:48:57.568574+0000","last_unstale":"2023-10-09T06:50:31.577591+0000","last_undegraded":"2023-10-09T06:50:31.577591+0000","last_fullsized":"2023-10-09T06:50:31.577591+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572981+0000","last_change":"2023-10-09T06:45:33.529820+0000","last_active":"2023-10-09T06:50:31.572981+0000","last_peered":"2023-10-09T06:50:31.572981+0000","last_clean":"2023-10-09T06:50:31.572981+0000","last_became_active":"2023-10-09T06:45:33.529430+0000","last_became_peered":"2023-10-09T06:45:33.529430+0000","last_unstale":"2023-10-09T06:50:31.572981+0000","last_undegraded":"2023-10-09T06:50:31.572981+0000","last_fullsized":"2023-10-09T06:50:31.572981+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":311,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572865+0000","last_change":"2023-10-09T06:45:33.530387+0000","last_active":"2023-10-09T06:50:31.572865+0000","last_peered":"2023-10-09T06:50:31.572865+0000","last_clean":"2023-10-09T06:50:31.572865+0000","last_became_active":"2023-10-09T06:45:33.530012+0000","last_became_peered":"2023-10-09T06:45:33.530012+0000","last_unstale":"2023-10-09T06:50:31.572865+0000","last_undegraded":"2023-10-09T06:50:31.572865+0000","last_fullsized":"2023-10-09T06:50:31.572865+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":75,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573025+0000","last_change":"2023-10-09T06:45:33.813820+0000","last_active":"2023-10-09T06:50:31.573025+0000","last_peered":"2023-10-09T06:50:31.573025+0000","last_clean":"2023-10-09T06:50:31.573025+0000","last_became_active":"2023-10-09T06:45:33.536995+0000","last_became_peered":"2023-10-09T06:45:33.536995+0000","last_unstale":"2023-10-09T06:50:31.573025+0000","last_undegraded":"2023-10-09T06:50:31.573025+0000","last_fullsized":"2023-10-09T06:50:31.573025+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":350,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573272+0000","last_change":"2023-10-09T06:45:33.534114+0000","last_active":"2023-10-09T06:50:31.573272+0000","last_peered":"2023-10-09T06:50:31.573272+0000","last_clean":"2023-10-09T06:50:31.573272+0000","last_became_active":"2023-10-09T06:45:33.533792+0000","last_became_peered":"2023-10-09T06:45:33.533792+0000","last_unstale":"2023-10-09T06:50:31.573272+0000","last_undegraded":"2023-10-09T06:50:31.573272+0000","last_fullsized":"2023-10-09T06:50:31.573272+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":58,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569026+0000","last_change":"2023-10-09T06:45:33.544298+0000","last_active":"2023-10-09T06:50:31.569026+0000","last_peered":"2023-10-09T06:50:31.569026+0000","last_clean":"2023-10-09T06:50:31.569026+0000","last_became_active":"2023-10-09T06:45:33.543919+0000","last_became_peered":"2023-10-09T06:45:33.543919+0000","last_unstale":"2023-10-09T06:50:31.569026+0000","last_undegraded":"2023-10-09T06:50:31.569026+0000","last_fullsized":"2023-10-09T06:50:31.569026+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573382+0000","last_change":"2023-10-09T06:49:20.585094+0000","last_active":"2023-10-09T06:50:31.573382+0000","last_peered":"2023-10-09T06:50:31.573382+0000","last_clean":"2023-10-09T06:50:31.573382+0000","last_became_active":"2023-10-09T06:49:20.584741+0000","last_became_peered":"2023-10-09T06:49:20.584741+0000","last_unstale":"2023-10-09T06:50:31.573382+0000","last_undegraded":"2023-10-09T06:50:31.573382+0000","last_fullsized":"2023-10-09T06:50:31.573382+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573978+0000","last_change":"2023-10-09T06:49:20.588455+0000","last_active":"2023-10-09T06:50:31.573978+0000","last_peered":"2023-10-09T06:50:31.573978+0000","last_clean":"2023-10-09T06:50:31.573978+0000","last_became_active":"2023-10-09T06:49:20.587543+0000","last_became_peered":"2023-10-09T06:49:20.587543+0000","last_unstale":"2023-10-09T06:50:31.573978+0000","last_undegraded":"2023-10-09T06:50:31.573978+0000","last_fullsized":"2023-10-09T06:50:31.573978+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":359,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573840+0000","last_change":"2023-10-09T06:45:33.533939+0000","last_active":"2023-10-09T06:50:31.573840+0000","last_peered":"2023-10-09T06:50:31.573840+0000","last_clean":"2023-10-09T06:50:31.573840+0000","last_became_active":"2023-10-09T06:45:33.533600+0000","last_became_peered":"2023-10-09T06:45:33.533600+0000","last_unstale":"2023-10-09T06:50:31.573840+0000","last_undegraded":"2023-10-09T06:50:31.573840+0000","last_fullsized":"2023-10-09T06:50:31.573840+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":317,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.568961+0000","last_change":"2023-10-09T06:45:33.534264+0000","last_active":"2023-10-09T06:50:31.568961+0000","last_peered":"2023-10-09T06:50:31.568961+0000","last_clean":"2023-10-09T06:50:31.568961+0000","last_became_active":"2023-10-09T06:45:33.533903+0000","last_became_peered":"2023-10-09T06:45:33.533903+0000","last_unstale":"2023-10-09T06:50:31.568961+0000","last_undegraded":"2023-10-09T06:50:31.568961+0000","last_fullsized":"2023-10-09T06:50:31.568961+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.574240+0000","last_change":"2023-10-09T06:45:33.529126+0000","last_active":"2023-10-09T06:50:31.574240+0000","last_peered":"2023-10-09T06:50:31.574240+0000","last_clean":"2023-10-09T06:50:31.574240+0000","last_became_active":"2023-10-09T06:45:33.528749+0000","last_became_peered":"2023-10-09T06:45:33.528749+0000","last_unstale":"2023-10-09T06:50:31.574240+0000","last_undegraded":"2023-10-09T06:50:31.574240+0000","last_fullsized":"2023-10-09T06:50:31.574240+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"341'263","reported_seq":1432,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:34.920142+0000","last_change":"2023-10-09T06:50:32.576339+0000","last_active":"2023-10-09T06:50:34.920142+0000","last_peered":"2023-10-09T06:50:34.920142+0000","last_clean":"2023-10-09T06:50:34.920142+0000","last_became_active":"2023-10-09T06:45:37.930357+0000","last_became_peered":"2023-10-09T06:45:37.930357+0000","last_unstale":"2023-10-09T06:50:34.920142+0000","last_undegraded":"2023-10-09T06:50:34.920142+0000","last_fullsized":"2023-10-09T06:50:34.920142+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":263,"log_dups_size":0,"ondisk_log_size":263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2048,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1922,"num_read_kb":4045,"num_write":236,"num_write_kb":2341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":221,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569431+0000","last_change":"2023-10-09T06:45:33.535684+0000","last_active":"2023-10-09T06:50:31.569431+0000","last_peered":"2023-10-09T06:50:31.569431+0000","last_clean":"2023-10-09T06:50:31.569431+0000","last_became_active":"2023-10-09T06:45:33.535320+0000","last_became_peered":"2023-10-09T06:45:33.535320+0000","last_unstale":"2023-10-09T06:50:31.569431+0000","last_undegraded":"2023-10-09T06:50:31.569431+0000","last_fullsized":"2023-10-09T06:50:31.569431+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573327+0000","last_change":"2023-10-09T06:49:20.584923+0000","last_active":"2023-10-09T06:50:31.573327+0000","last_peered":"2023-10-09T06:50:31.573327+0000","last_clean":"2023-10-09T06:50:31.573327+0000","last_became_active":"2023-10-09T06:49:20.584322+0000","last_became_peered":"2023-10-09T06:49:20.584322+0000","last_unstale":"2023-10-09T06:50:31.573327+0000","last_undegraded":"2023-10-09T06:50:31.573327+0000","last_fullsized":"2023-10-09T06:50:31.573327+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"341'97","reported_seq":194,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:34.818667+0000","last_change":"2023-10-09T06:50:32.574382+0000","last_active":"2023-10-09T06:50:34.818667+0000","last_peered":"2023-10-09T06:50:34.818667+0000","last_clean":"2023-10-09T06:50:34.818667+0000","last_became_active":"2023-10-09T06:45:33.543602+0000","last_became_peered":"2023-10-09T06:45:33.543602+0000","last_unstale":"2023-10-09T06:50:34.818667+0000","last_undegraded":"2023-10-09T06:50:34.818667+0000","last_fullsized":"2023-10-09T06:50:34.818667+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1007,"num_read_kb":1573,"num_write":85,"num_write_kb":1401,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573167+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:50:31.573167+0000","last_peered":"2023-10-09T06:50:31.573167+0000","last_clean":"2023-10-09T06:50:31.573167+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:50:31.573167+0000","last_undegraded":"2023-10-09T06:50:31.573167+0000","last_fullsized":"2023-10-09T06:50:31.573167+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.578113+0000","last_change":"2023-10-09T06:48:57.565279+0000","last_active":"2023-10-09T06:50:31.578113+0000","last_peered":"2023-10-09T06:50:31.578113+0000","last_clean":"2023-10-09T06:50:31.578113+0000","last_became_active":"2023-10-09T06:48:57.564796+0000","last_became_peered":"2023-10-09T06:48:57.564796+0000","last_unstale":"2023-10-09T06:50:31.578113+0000","last_undegraded":"2023-10-09T06:50:31.578113+0000","last_fullsized":"2023-10-09T06:50:31.578113+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"341'168","reported_seq":650,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:35.053027+0000","last_change":"2023-10-09T06:50:32.634829+0000","last_active":"2023-10-09T06:50:35.053027+0000","last_peered":"2023-10-09T06:50:35.053027+0000","last_clean":"2023-10-09T06:50:35.053027+0000","last_became_active":"2023-10-09T06:45:36.692765+0000","last_became_peered":"2023-10-09T06:45:36.692765+0000","last_unstale":"2023-10-09T06:50:35.053027+0000","last_undegraded":"2023-10-09T06:50:35.053027+0000","last_fullsized":"2023-10-09T06:50:35.053027+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":168,"log_dups_size":0,"ondisk_log_size":168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194320,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":863,"num_read_kb":1074,"num_write":152,"num_write_kb":4398,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577553+0000","last_change":"2023-10-09T06:45:33.532462+0000","last_active":"2023-10-09T06:50:31.577553+0000","last_peered":"2023-10-09T06:50:31.577553+0000","last_clean":"2023-10-09T06:50:31.577553+0000","last_became_active":"2023-10-09T06:45:33.531726+0000","last_became_peered":"2023-10-09T06:45:33.531726+0000","last_unstale":"2023-10-09T06:50:31.577553+0000","last_undegraded":"2023-10-09T06:50:31.577553+0000","last_fullsized":"2023-10-09T06:50:31.577553+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":212,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570148+0000","last_change":"2023-10-09T06:49:20.583160+0000","last_active":"2023-10-09T06:50:31.570148+0000","last_peered":"2023-10-09T06:50:31.570148+0000","last_clean":"2023-10-09T06:50:31.570148+0000","last_became_active":"2023-10-09T06:49:20.582682+0000","last_became_peered":"2023-10-09T06:49:20.582682+0000","last_unstale":"2023-10-09T06:50:31.570148+0000","last_undegraded":"2023-10-09T06:50:31.570148+0000","last_fullsized":"2023-10-09T06:50:31.570148+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"341'208","reported_seq":675,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:34.401601+0000","last_change":"2023-10-09T06:50:32.576064+0000","last_active":"2023-10-09T06:50:34.401601+0000","last_peered":"2023-10-09T06:50:34.401601+0000","last_clean":"2023-10-09T06:50:34.401601+0000","last_became_active":"2023-10-09T06:45:37.929757+0000","last_became_peered":"2023-10-09T06:45:37.929757+0000","last_unstale":"2023-10-09T06:50:34.401601+0000","last_undegraded":"2023-10-09T06:50:34.401601+0000","last_fullsized":"2023-10-09T06:50:34.401601+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":371,"num_read_kb":2600,"num_write":211,"num_write_kb":1812,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"1.0","version":"84'139","reported_seq":636,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.568816+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:50:31.568816+0000","last_peered":"2023-10-09T06:50:31.568816+0000","last_clean":"2023-10-09T06:50:31.568816+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:50:31.568816+0000","last_undegraded":"2023-10-09T06:50:31.568816+0000","last_fullsized":"2023-10-09T06:50:31.568816+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":332,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.577986+0000","last_change":"2023-10-09T06:45:33.532290+0000","last_active":"2023-10-09T06:50:31.577986+0000","last_peered":"2023-10-09T06:50:31.577986+0000","last_clean":"2023-10-09T06:50:31.577986+0000","last_became_active":"2023-10-09T06:45:33.531814+0000","last_became_peered":"2023-10-09T06:45:33.531814+0000","last_unstale":"2023-10-09T06:50:31.577986+0000","last_undegraded":"2023-10-09T06:50:31.577986+0000","last_fullsized":"2023-10-09T06:50:31.577986+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":314,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573839+0000","last_change":"2023-10-09T06:42:32.498269+0000","last_active":"2023-10-09T06:50:31.573839+0000","last_peered":"2023-10-09T06:50:31.573839+0000","last_clean":"2023-10-09T06:50:31.573839+0000","last_became_active":"2023-10-09T06:42:32.497812+0000","last_became_peered":"2023-10-09T06:42:32.497812+0000","last_unstale":"2023-10-09T06:50:31.573839+0000","last_undegraded":"2023-10-09T06:50:31.573839+0000","last_fullsized":"2023-10-09T06:50:31.573839+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"341'161","reported_seq":571,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:32.714643+0000","last_change":"2023-10-09T06:50:32.575841+0000","last_active":"2023-10-09T06:50:32.714643+0000","last_peered":"2023-10-09T06:50:32.714643+0000","last_clean":"2023-10-09T06:50:32.714643+0000","last_became_active":"2023-10-09T06:45:33.527346+0000","last_became_peered":"2023-10-09T06:45:33.527346+0000","last_unstale":"2023-10-09T06:50:32.714643+0000","last_undegraded":"2023-10-09T06:50:32.714643+0000","last_fullsized":"2023-10-09T06:50:32.714643+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":962,"num_read_kb":2402,"num_write":135,"num_write_kb":1368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572803+0000","last_change":"2023-10-09T06:49:20.591229+0000","last_active":"2023-10-09T06:50:31.572803+0000","last_peered":"2023-10-09T06:50:31.572803+0000","last_clean":"2023-10-09T06:50:31.572803+0000","last_became_active":"2023-10-09T06:49:20.590718+0000","last_became_peered":"2023-10-09T06:49:20.590718+0000","last_unstale":"2023-10-09T06:50:31.572803+0000","last_undegraded":"2023-10-09T06:50:31.572803+0000","last_fullsized":"2023-10-09T06:50:31.572803+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570002+0000","last_change":"2023-10-09T06:49:20.585713+0000","last_active":"2023-10-09T06:50:31.570002+0000","last_peered":"2023-10-09T06:50:31.570002+0000","last_clean":"2023-10-09T06:50:31.570002+0000","last_became_active":"2023-10-09T06:49:20.585324+0000","last_became_peered":"2023-10-09T06:49:20.585324+0000","last_unstale":"2023-10-09T06:50:31.570002+0000","last_undegraded":"2023-10-09T06:50:31.570002+0000","last_fullsized":"2023-10-09T06:50:31.570002+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"341'129","reported_seq":449,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:34.826010+0000","last_change":"2023-10-09T06:50:32.576091+0000","last_active":"2023-10-09T06:50:34.826010+0000","last_peered":"2023-10-09T06:50:34.826010+0000","last_clean":"2023-10-09T06:50:34.826010+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:50:34.826010+0000","last_undegraded":"2023-10-09T06:50:34.826010+0000","last_fullsized":"2023-10-09T06:50:34.826010+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4137,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":930,"num_read_kb":1808,"num_write":110,"num_write_kb":1074,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573538+0000","last_change":"2023-10-09T06:45:05.135956+0000","last_active":"2023-10-09T06:50:31.573538+0000","last_peered":"2023-10-09T06:50:31.573538+0000","last_clean":"2023-10-09T06:50:31.573538+0000","last_became_active":"2023-10-09T06:45:05.135547+0000","last_became_peered":"2023-10-09T06:45:05.135547+0000","last_unstale":"2023-10-09T06:50:31.573538+0000","last_undegraded":"2023-10-09T06:50:31.573538+0000","last_fullsized":"2023-10-09T06:50:31.573538+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":353,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570814+0000","last_change":"2023-10-09T06:49:20.583253+0000","last_active":"2023-10-09T06:50:31.570814+0000","last_peered":"2023-10-09T06:50:31.570814+0000","last_clean":"2023-10-09T06:50:31.570814+0000","last_became_active":"2023-10-09T06:49:20.582889+0000","last_became_peered":"2023-10-09T06:49:20.582889+0000","last_unstale":"2023-10-09T06:50:31.570814+0000","last_undegraded":"2023-10-09T06:50:31.570814+0000","last_fullsized":"2023-10-09T06:50:31.570814+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":163,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.570026+0000","last_change":"2023-10-09T06:45:33.940589+0000","last_active":"2023-10-09T06:50:31.570026+0000","last_peered":"2023-10-09T06:50:31.570026+0000","last_clean":"2023-10-09T06:50:31.570026+0000","last_became_active":"2023-10-09T06:45:33.541861+0000","last_became_peered":"2023-10-09T06:45:33.541861+0000","last_unstale":"2023-10-09T06:50:31.570026+0000","last_undegraded":"2023-10-09T06:50:31.570026+0000","last_fullsized":"2023-10-09T06:50:31.570026+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"341'156","reported_seq":886,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:35.022460+0000","last_change":"2023-10-09T06:50:32.635056+0000","last_active":"2023-10-09T06:50:35.022460+0000","last_peered":"2023-10-09T06:50:35.022460+0000","last_clean":"2023-10-09T06:50:35.022460+0000","last_became_active":"2023-10-09T06:45:36.693034+0000","last_became_peered":"2023-10-09T06:45:36.693034+0000","last_unstale":"2023-10-09T06:50:35.022460+0000","last_undegraded":"2023-10-09T06:50:35.022460+0000","last_fullsized":"2023-10-09T06:50:35.022460+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4096,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":889,"num_read_kb":1094,"num_write":106,"num_write_kb":255,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.568897+0000","last_change":"2023-10-09T06:48:57.578038+0000","last_active":"2023-10-09T06:50:31.568897+0000","last_peered":"2023-10-09T06:50:31.568897+0000","last_clean":"2023-10-09T06:50:31.568897+0000","last_became_active":"2023-10-09T06:48:57.577546+0000","last_became_peered":"2023-10-09T06:48:57.577546+0000","last_unstale":"2023-10-09T06:50:31.568897+0000","last_undegraded":"2023-10-09T06:50:31.568897+0000","last_fullsized":"2023-10-09T06:50:31.568897+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"341'235","reported_seq":1109,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-09T06:50:34.537808+0000","last_change":"2023-10-09T06:50:32.578935+0000","last_active":"2023-10-09T06:50:34.537808+0000","last_peered":"2023-10-09T06:50:34.537808+0000","last_clean":"2023-10-09T06:50:34.537808+0000","last_became_active":"2023-10-09T06:45:36.687770+0000","last_became_peered":"2023-10-09T06:45:36.687770+0000","last_unstale":"2023-10-09T06:50:34.537808+0000","last_undegraded":"2023-10-09T06:50:34.537808+0000","last_fullsized":"2023-10-09T06:50:34.537808+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":235,"log_dups_size":0,"ondisk_log_size":235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1615,"num_read_kb":1816,"num_write":195,"num_write_kb":1312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.6","version":"60'1","reported_seq":82,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573959+0000","last_change":"2023-10-09T06:44:32.353758+0000","last_active":"2023-10-09T06:50:31.573959+0000","last_peered":"2023-10-09T06:50:31.573959+0000","last_clean":"2023-10-09T06:50:31.573959+0000","last_became_active":"2023-10-09T06:44:32.353240+0000","last_became_peered":"2023-10-09T06:44:32.353240+0000","last_unstale":"2023-10-09T06:50:31.573959+0000","last_undegraded":"2023-10-09T06:50:31.573959+0000","last_fullsized":"2023-10-09T06:50:31.573959+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":335,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573612+0000","last_change":"2023-10-09T06:42:32.476668+0000","last_active":"2023-10-09T06:50:31.573612+0000","last_peered":"2023-10-09T06:50:31.573612+0000","last_clean":"2023-10-09T06:50:31.573612+0000","last_became_active":"2023-10-09T06:42:32.476053+0000","last_became_peered":"2023-10-09T06:42:32.476053+0000","last_unstale":"2023-10-09T06:50:31.573612+0000","last_undegraded":"2023-10-09T06:50:31.573612+0000","last_fullsized":"2023-10-09T06:50:31.573612+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":98,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573791+0000","last_change":"2023-10-09T06:48:57.563675+0000","last_active":"2023-10-09T06:50:31.573791+0000","last_peered":"2023-10-09T06:50:31.573791+0000","last_clean":"2023-10-09T06:50:31.573791+0000","last_became_active":"2023-10-09T06:48:57.563237+0000","last_became_peered":"2023-10-09T06:48:57.563237+0000","last_unstale":"2023-10-09T06:50:31.573791+0000","last_undegraded":"2023-10-09T06:50:31.573791+0000","last_fullsized":"2023-10-09T06:50:31.573791+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":322,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573426+0000","last_change":"2023-10-09T06:48:57.566453+0000","last_active":"2023-10-09T06:50:31.573426+0000","last_peered":"2023-10-09T06:50:31.573426+0000","last_clean":"2023-10-09T06:50:31.573426+0000","last_became_active":"2023-10-09T06:48:57.565889+0000","last_became_peered":"2023-10-09T06:48:57.565889+0000","last_unstale":"2023-10-09T06:50:31.573426+0000","last_undegraded":"2023-10-09T06:50:31.573426+0000","last_fullsized":"2023-10-09T06:50:31.573426+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":174,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573702+0000","last_change":"2023-10-09T06:42:32.593843+0000","last_active":"2023-10-09T06:50:31.573702+0000","last_peered":"2023-10-09T06:50:31.573702+0000","last_clean":"2023-10-09T06:50:31.573702+0000","last_became_active":"2023-10-09T06:42:32.483691+0000","last_became_peered":"2023-10-09T06:42:32.483691+0000","last_unstale":"2023-10-09T06:50:31.573702+0000","last_undegraded":"2023-10-09T06:50:31.573702+0000","last_fullsized":"2023-10-09T06:50:31.573702+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573365+0000","last_change":"2023-10-09T06:49:20.584379+0000","last_active":"2023-10-09T06:50:31.573365+0000","last_peered":"2023-10-09T06:50:31.573365+0000","last_clean":"2023-10-09T06:50:31.573365+0000","last_became_active":"2023-10-09T06:49:20.582540+0000","last_became_peered":"2023-10-09T06:49:20.582540+0000","last_unstale":"2023-10-09T06:50:31.573365+0000","last_undegraded":"2023-10-09T06:50:31.573365+0000","last_fullsized":"2023-10-09T06:50:31.573365+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":187,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.573615+0000","last_change":"2023-10-09T06:45:33.762363+0000","last_active":"2023-10-09T06:50:31.573615+0000","last_peered":"2023-10-09T06:50:31.573615+0000","last_clean":"2023-10-09T06:50:31.573615+0000","last_became_active":"2023-10-09T06:45:33.530125+0000","last_became_peered":"2023-10-09T06:45:33.530125+0000","last_unstale":"2023-10-09T06:50:31.573615+0000","last_undegraded":"2023-10-09T06:50:31.573615+0000","last_fullsized":"2023-10-09T06:50:31.573615+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572737+0000","last_change":"2023-10-09T06:49:20.582679+0000","last_active":"2023-10-09T06:50:31.572737+0000","last_peered":"2023-10-09T06:50:31.572737+0000","last_clean":"2023-10-09T06:50:31.572737+0000","last_became_active":"2023-10-09T06:49:20.582285+0000","last_became_peered":"2023-10-09T06:49:20.582285+0000","last_unstale":"2023-10-09T06:50:31.572737+0000","last_undegraded":"2023-10-09T06:50:31.572737+0000","last_fullsized":"2023-10-09T06:50:31.572737+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":371,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.569811+0000","last_change":"2023-10-09T06:45:33.542948+0000","last_active":"2023-10-09T06:50:31.569811+0000","last_peered":"2023-10-09T06:50:31.569811+0000","last_clean":"2023-10-09T06:50:31.569811+0000","last_became_active":"2023-10-09T06:45:33.542464+0000","last_became_peered":"2023-10-09T06:45:33.542464+0000","last_unstale":"2023-10-09T06:50:31.569811+0000","last_undegraded":"2023-10-09T06:50:31.569811+0000","last_fullsized":"2023-10-09T06:50:31.569811+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":27,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572624+0000","last_change":"2023-10-09T06:49:20.586102+0000","last_active":"2023-10-09T06:50:31.572624+0000","last_peered":"2023-10-09T06:50:31.572624+0000","last_clean":"2023-10-09T06:50:31.572624+0000","last_became_active":"2023-10-09T06:49:20.585465+0000","last_became_peered":"2023-10-09T06:49:20.585465+0000","last_unstale":"2023-10-09T06:50:31.572624+0000","last_undegraded":"2023-10-09T06:50:31.572624+0000","last_fullsized":"2023-10-09T06:50:31.572624+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":362,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.578710+0000","last_change":"2023-10-09T06:48:57.573215+0000","last_active":"2023-10-09T06:50:31.578710+0000","last_peered":"2023-10-09T06:50:31.578710+0000","last_clean":"2023-10-09T06:50:31.578710+0000","last_became_active":"2023-10-09T06:48:57.569057+0000","last_became_peered":"2023-10-09T06:48:57.569057+0000","last_unstale":"2023-10-09T06:50:31.578710+0000","last_undegraded":"2023-10-09T06:50:31.578710+0000","last_fullsized":"2023-10-09T06:50:31.578710+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":377,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-09T06:50:31.572724+0000","last_change":"2023-10-09T06:48:57.578851+0000","last_active":"2023-10-09T06:50:31.572724+0000","last_peered":"2023-10-09T06:50:31.572724+0000","last_clean":"2023-10-09T06:50:31.572724+0000","last_became_active":"2023-10-09T06:48:57.578423+0000","last_became_peered":"2023-10-09T06:48:57.578423+0000","last_unstale":"2023-10-09T06:50:31.572724+0000","last_undegraded":"2023-10-09T06:50:31.572724+0000","last_fullsized":"2023-10-09T06:50:31.572724+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":31934,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4204661,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":8559,"num_read_kb":16412,"num_write":1230,"num_write_kb":13961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":262,"num_bytes_recovered":4194399,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21082112,"data_stored":21014912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":1417,"ondisk_log_size":1417,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430831,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905604,"kb_used_data":3576,"kb_used_omap":0,"kb_used_meta":902016,"kb_avail":92839548,"statfs":{"total":95995035648,"available":95067697152,"internally_reserved":0,"allocated":3661824,"data_stored":3024458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923664384},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":2.081,"15min":1.4710000000000001},"min":{"1min":0.435,"5min":0.224,"15min":0.224},"max":{"1min":1.355,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.1259999999999999,"15min":1.4259999999999999},"min":{"1min":0.48099999999999998,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":1.2609999999999999,"5min":91.715999999999994,"15min":91.715999999999994},"last":0.76400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.212,"15min":1.466},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":2.0910000000000002,"5min":6.3559999999999999,"15min":13.526999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.877,"5min":1.2829999999999999,"15min":1.3720000000000001},"min":{"1min":0.45200000000000001,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":2.1110000000000002,"5min":6.4359999999999999,"15min":13.922000000000001},"last":0.72699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:50:13 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.819,"15min":1.6830000000000001},"min":{"1min":0.48599999999999999,"5min":0.35999999999999999,"15min":0.29099999999999998},"max":{"1min":5.5,"5min":9.0679999999999996,"15min":15.957000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.27,"15min":1.292},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":1.579,"5min":6.3319999999999999,"15min":6.3319999999999999},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:49:48 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.8979999999999999,"15min":2.278},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.30499999999999999},"max":{"1min":5.8099999999999996,"5min":9.0449999999999999,"15min":32.715000000000003},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.323,"15min":1.8260000000000001},"min":{"1min":0.45400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.0739999999999998,"5min":6.4690000000000003,"15min":32.728999999999999},"last":0.80400000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.782,"15min":1.2729999999999999},"min":{"1min":0.63200000000000001,"5min":0.27200000000000002,"15min":0.26700000000000002},"max":{"1min":1.319,"5min":12.711,"15min":12.711},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.008,"5min":2.6859999999999999,"15min":1.893},"min":{"1min":0.47099999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.5229999999999999,"5min":11.369999999999999,"15min":16.064},"last":0.78100000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.319,"15min":1.2070000000000001},"min":{"1min":0.49299999999999999,"5min":0.35099999999999998,"15min":0.34799999999999998},"max":{"1min":1.4059999999999999,"5min":6.3049999999999997,"15min":10.680999999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.2629999999999999,"15min":1.1919999999999999},"min":{"1min":0.63100000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.3169999999999999,"5min":12.675000000000001,"15min":12.675000000000001},"last":0.67100000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.885,"15min":1.419},"min":{"1min":0.70599999999999996,"5min":0.32900000000000001,"15min":0.32200000000000001},"max":{"1min":1.349,"5min":12.640000000000001,"15min":12.640000000000001},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.0640000000000001,"5min":2.7130000000000001,"15min":1.9279999999999999},"min":{"1min":0.73699999999999999,"5min":0.42999999999999999,"15min":0.378},"max":{"1min":1.373,"5min":14.613,"15min":16.024000000000001},"last":0.85599999999999998}]}]},{"osd":6,"up_from":42,"seq":180388627063,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896020,"kb_used_data":7676,"kb_used_omap":0,"kb_used_meta":888320,"kb_avail":92849132,"statfs":{"total":95995035648,"available":95077511168,"internally_reserved":0,"allocated":7860224,"data_stored":7228830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909639680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":4.4180000000000001,"15min":2.2749999999999999},"min":{"1min":0.38800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.4279999999999999,"5min":120.822,"15min":120.822},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.77900000000000003,"5min":4.7990000000000004,"15min":2.5070000000000001},"min":{"1min":0.49099999999999999,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":1.0920000000000001,"5min":120.47199999999999,"15min":120.47199999999999},"last":0.65800000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.26,"15min":1.5009999999999999},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.0869999999999997,"5min":5.6909999999999998,"15min":19.25},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.262,"15min":1.3839999999999999},"min":{"1min":0.497,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":2.8340000000000001,"5min":6.0369999999999999,"15min":7.952},"last":0.752}]},{"osd":2,"last update":"Mon Oct 9 06:50:20 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.26,"15min":2.4449999999999998},"min":{"1min":0.52100000000000002,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":1.5289999999999999,"5min":6.0780000000000003,"15min":271.976},"last":1.218},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.7190000000000001,"15min":2.661},"min":{"1min":0.56100000000000005,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":3.7639999999999998,"5min":11.59,"15min":271.863},"last":1.028}]},{"osd":3,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.2609999999999999,"15min":1.552},"min":{"1min":0.499,"5min":0.36099999999999999,"15min":0.32800000000000001},"max":{"1min":2.0859999999999999,"5min":6.0570000000000004,"15min":22.873999999999999},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.3300000000000001,"15min":1.702},"min":{"1min":0.65400000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.1230000000000002,"5min":5.7720000000000002,"15min":25.986999999999998},"last":0.85499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":2.4550000000000001,"15min":1.9850000000000001},"min":{"1min":0.44500000000000001,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":12.212999999999999,"5min":13.053000000000001,"15min":80.231999999999999},"last":1.258},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.085,"15min":1.9159999999999999},"min":{"1min":0.42299999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.0170000000000003,"5min":20.196000000000002,"15min":81.085999999999999},"last":1.133}]},{"osd":5,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.7649999999999999,"15min":1.389},"min":{"1min":0.55000000000000004,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":7.1609999999999996,"5min":11.558,"15min":11.558},"last":1.429},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.347,"15min":1.538},"min":{"1min":0.45800000000000002,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":5.0599999999999996,"5min":5.7400000000000002,"15min":11.526},"last":1.3029999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.4889999999999999,"15min":1.756},"min":{"1min":0.69899999999999995,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":3.9009999999999998,"5min":12.177,"15min":12.177},"last":1.5169999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.734,"15min":1.3660000000000001},"min":{"1min":0.77200000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":1.6850000000000001,"5min":9.5619999999999994,"15min":9.5619999999999994},"last":1.4019999999999999}]}]},{"osd":1,"up_from":0,"seq":399431958973,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122499,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952168,"kb_used_data":8340,"kb_used_omap":8,"kb_used_meta":943799,"kb_avail":92792984,"statfs":{"total":95995035648,"available":95020015616,"internally_reserved":0,"allocated":8540160,"data_stored":7323744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":966450432},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.71299999999999997,"15min":1.5269999999999999},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.187},"max":{"1min":1.4810000000000001,"5min":9.0670000000000002,"15min":25.084},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.64600000000000002,"15min":1.534},"min":{"1min":0.374,"5min":0.27800000000000002,"15min":0.20899999999999999},"max":{"1min":1.121,"5min":3.8929999999999998,"15min":25.036999999999999},"last":0.496}]},{"osd":2,"last update":"Mon Oct 9 06:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.78800000000000003,"15min":1.3140000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.20599999999999999},"max":{"1min":1.2709999999999999,"5min":9.1539999999999999,"15min":12.154},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.623,"15min":1.3140000000000001},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.215},"max":{"1min":1.1279999999999999,"5min":3.9489999999999998,"15min":15.351000000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":2.2730000000000001},"min":{"1min":0.39100000000000001,"5min":0.27300000000000002,"15min":0.216},"max":{"1min":0.90200000000000002,"5min":2.851,"15min":86.709999999999994},"last":0.371},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.69999999999999996,"15min":2.3330000000000002},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":1.274,"5min":3.9940000000000002,"15min":87.109999999999999},"last":0.52000000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.367,"15min":1.431},"min":{"1min":0.53600000000000003,"5min":0.35899999999999999,"15min":0.309},"max":{"1min":4.1109999999999998,"5min":9.5459999999999994,"15min":9.5459999999999994},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.399,"15min":1.474},"min":{"1min":0.45100000000000001,"5min":0.377,"15min":0.31},"max":{"1min":2.1070000000000002,"5min":11.948,"15min":11.948},"last":0.53500000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0549999999999999,"15min":1.4650000000000001},"min":{"1min":0.51900000000000002,"5min":0.36699999999999999,"15min":0.27800000000000002},"max":{"1min":3.7370000000000001,"5min":11.894,"15min":11.894},"last":0.438},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.95799999999999996,"15min":1.4430000000000001},"min":{"1min":0.46999999999999997,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":3.7810000000000001,"5min":4.6520000000000001,"15min":10.419},"last":0.66000000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.3500000000000001,"15min":1.5920000000000001},"min":{"1min":0.48899999999999999,"5min":0.432,"15min":0.33900000000000002},"max":{"1min":3.544,"5min":8.0619999999999994,"15min":10.377000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.117,"5min":1.2709999999999999,"15min":1.4650000000000001},"min":{"1min":0.54900000000000004,"5min":0.43099999999999999,"15min":0.379},"max":{"1min":3.4750000000000001,"5min":6.1159999999999997,"15min":9.4510000000000005},"last":0.64100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:49:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.492,"15min":1.778},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.40100000000000002},"max":{"1min":3.419,"5min":5.9740000000000002,"15min":13.259},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.379,"15min":1.6679999999999999},"min":{"1min":0.53400000000000003,"5min":0.42399999999999999,"15min":0.35299999999999998},"max":{"1min":3.4340000000000002,"5min":5.798,"15min":13.428000000000001},"last":0.72699999999999998}]}]},{"osd":2,"up_from":0,"seq":420906795447,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":442381631921,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019529,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902320,"kb_used_data":7128,"kb_used_omap":0,"kb_used_meta":895168,"kb_avail":92842832,"statfs":{"total":95995035648,"available":95071059968,"internally_reserved":0,"allocated":7299072,"data_stored":6649482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916652032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":3.871,"15min":2.8940000000000001},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.249},"max":{"1min":18.120999999999999,"5min":91.683999999999997,"15min":91.683999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.6850000000000001,"5min":4.1059999999999999,"15min":3},"min":{"1min":0.47299999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":18.158000000000001,"5min":91.653999999999996,"15min":91.653999999999996},"last":0.52600000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:49:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":1.3440000000000001,"15min":1.3100000000000001},"min":{"1min":0.499,"5min":0.34799999999999998,"15min":0.28399999999999997},"max":{"1min":6.4610000000000003,"5min":7.0659999999999998,"15min":10.553000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.006,"15min":1.286},"min":{"1min":0.373,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":4.4779999999999998,"5min":4.4779999999999998,"15min":9.1129999999999995},"last":0.747}]},{"osd":2,"last update":"Mon Oct 9 06:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.6830000000000001,"15min":3.2810000000000001},"min":{"1min":0.623,"5min":0.38400000000000001,"15min":0.28599999999999998},"max":{"1min":1.319,"5min":10.739000000000001,"15min":231.393},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.1719999999999999,"15min":3.1059999999999999},"min":{"1min":0.48199999999999998,"5min":0.34100000000000003,"15min":0.29999999999999999},"max":{"1min":2.9169999999999998,"5min":8.7149999999999999,"15min":231.17099999999999},"last":0.94999999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:49:44 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.155,"15min":4.2560000000000002},"min":{"1min":0.376,"5min":0.376,"15min":0.34100000000000003},"max":{"1min":4.4370000000000003,"5min":8.6509999999999998,"15min":234.62200000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.234,"5min":1.4710000000000001,"15min":4.391},"min":{"1min":0.57899999999999996,"5min":0.40400000000000003,"15min":0.33900000000000002},"max":{"1min":4.5979999999999999,"5min":8.6340000000000003,"15min":234.673},"last":0.92000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:50:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.4319999999999999,"15min":1.3009999999999999},"min":{"1min":0.47999999999999998,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":4.5410000000000004,"5min":7.0330000000000004,"15min":22.068000000000001},"last":1.008},{"interface":"front","average":{"1min":1.456,"5min":1.6950000000000001,"15min":1.3999999999999999},"min":{"1min":0.35499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.8609999999999998,"5min":10.707000000000001,"15min":13.095000000000001},"last":0.89700000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:49:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.633,"15min":1.708},"min":{"1min":0.55300000000000005,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":12.507999999999999,"5min":13.163,"15min":13.163},"last":1.1779999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.044,"15min":1.4379999999999999},"min":{"1min":0.57999999999999996,"5min":0.42099999999999999,"15min":0.32300000000000001},"max":{"1min":9.5719999999999992,"5min":10.138999999999999,"15min":10.138999999999999},"last":0.96999999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:49:36 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.5289999999999999,"15min":1.8420000000000001},"min":{"1min":0.498,"5min":0.312,"15min":0.312},"max":{"1min":10.688000000000001,"5min":10.688000000000001,"15min":10.688000000000001},"last":1.4099999999999999},{"interface":"front","average":{"1min":1.992,"5min":1.52,"15min":1.3560000000000001},"min":{"1min":0.435,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.9529999999999994,"5min":8.9529999999999994,"15min":8.9529999999999994},"last":0.99399999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823296,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4247552,"data_stored":4214975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4210688,"data_stored":4204560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4210688,"data_stored":4198476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":8293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:50:40.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:50:40.988 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:50:41.017 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:50:41.019 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.019 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.019 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.019 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.020 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.020 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.020 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.020 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:50:41.052 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:41.086 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:41.189 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:41.212 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:41.316 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:41.336 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:41.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:41.448 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:41] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:50:41.463 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:41.566 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:41.593 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:41.594 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: cluster 2023-10-09T06:50:40.179075+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: cluster 2023-10-09T06:50:40.449839+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: audit 2023-10-09T06:50:40.818047+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.125:0/3022626790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:41.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: cluster 2023-10-09T06:50:41.190434+0000 mon.a (mon.0) 1722 : cluster 0 osdmap e348: 8 total, 8 up, 4 in 2023-10-09T06:50:41.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: cluster 2023-10-09T06:50:41.193800+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:41.595 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:41 smithi155 bash[59495]: audit 2023-10-09T06:50:41.316697+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:50:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:41.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: cluster 2023-10-09T06:50:40.179075+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.675 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: cluster 2023-10-09T06:50:40.449839+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: audit 2023-10-09T06:50:40.818047+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.125:0/3022626790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:41.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: cluster 2023-10-09T06:50:41.190434+0000 mon.a (mon.0) 1722 : cluster 0 osdmap e348: 8 total, 8 up, 4 in 2023-10-09T06:50:41.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: cluster 2023-10-09T06:50:41.193800+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:41.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[77299]: audit 2023-10-09T06:50:41.316697+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:50:41.698 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:41.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: cluster 2023-10-09T06:50:40.179075+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: cluster 2023-10-09T06:50:40.449839+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e347 with expected crc 2023-10-09T06:50:41.709 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: audit 2023-10-09T06:50:40.818047+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.125:0/3022626790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:41.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: cluster 2023-10-09T06:50:41.190434+0000 mon.a (mon.0) 1722 : cluster 0 osdmap e348: 8 total, 8 up, 4 in 2023-10-09T06:50:41.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: cluster 2023-10-09T06:50:41.193800+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:41.710 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:41 smithi125 bash[80002]: audit 2023-10-09T06:50:41.316697+0000 mon.a (mon.0) 1723 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:50:41.722 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:41.825 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:41.853 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:41.957 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:41.986 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:41.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:42.090 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.118 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:42.204 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:50:42.209 INFO:tasks.workunit.client.0.smithi125.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:50:42.221 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg 2023-10-09T06:50:42.222 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:42.252 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:42.357 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:42.360 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:42.364 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd remove testimg_import 2023-10-09T06:50:42.382 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:42.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:41.189159+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:41.500456+0000 mgr.y (mgr.25723) 1390 : cluster 0 pgmap v1538: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 207 KiB/s rd, 20 KiB/s wr, 278 op/s 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:41.696019+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:42.186575+0000 mon.a (mon.0) 1724 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:42.186629+0000 mon.a (mon.0) 1725 : cluster 1 Cluster is now healthy 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: audit 2023-10-09T06:50:42.193982+0000 mon.a (mon.0) 1726 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:42.194056+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e349: 8 total, 8 up, 4 in 2023-10-09T06:50:42.489 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[77299]: cluster 2023-10-09T06:50:42.197328+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:42.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:41.189159+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:41.500456+0000 mgr.y (mgr.25723) 1390 : cluster 0 pgmap v1538: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 207 KiB/s rd, 20 KiB/s wr, 278 op/s 2023-10-09T06:50:42.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:41.696019+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.490 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:42.186575+0000 mon.a (mon.0) 1724 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:50:42.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:42.186629+0000 mon.a (mon.0) 1725 : cluster 1 Cluster is now healthy 2023-10-09T06:50:42.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: audit 2023-10-09T06:50:42.193982+0000 mon.a (mon.0) 1726 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:50:42.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:42.194056+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e349: 8 total, 8 up, 4 in 2023-10-09T06:50:42.491 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:42 smithi125 bash[80002]: cluster 2023-10-09T06:50:42.197328+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:42.494 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:42.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ tiered=0 2023-10-09T06:50:42.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ ceph osd dump 2023-10-09T06:50:42.499 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep ^pool 2023-10-09T06:50:42.500 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep 'rbd' 2023-10-09T06:50:42.500 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep tier 2023-10-09T06:50:42.514 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:42.617 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:41.189159+0000 osd.4 (osd.4) 294 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:41.500456+0000 mgr.y (mgr.25723) 1390 : cluster 0 pgmap v1538: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 207 KiB/s rd, 20 KiB/s wr, 278 op/s 2023-10-09T06:50:42.630 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:41.696019+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e348 with expected crc 2023-10-09T06:50:42.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:42.186575+0000 mon.a (mon.0) 1724 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:50:42.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:42.186629+0000 mon.a (mon.0) 1725 : cluster 1 Cluster is now healthy 2023-10-09T06:50:42.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: audit 2023-10-09T06:50:42.193982+0000 mon.a (mon.0) 1726 : audit 1 from='client.? 172.21.15.125:0/184477337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:50:42.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:42.194056+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e349: 8 total, 8 up, 4 in 2023-10-09T06:50:42.631 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:42 smithi155 bash[59495]: cluster 2023-10-09T06:50:42.197328+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:42.647 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:42.750 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.778 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:42.882 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:42.909 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:42.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:43.013 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:43.144 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:43.169 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:43.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:43.273 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:43.305 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:43.409 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.438 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:43.542 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:43.553 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:43 smithi155 bash[59495]: cluster 2023-10-09T06:50:42.796289+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:43.570 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:50:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:50:43.603 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_245068/sparse1 2023-10-09T06:50:43.611 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:50:43.612 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:50:43.612 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00687546 s, 153 MB/s 2023-10-09T06:50:43.612 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_245068/sparse2 2023-10-09T06:50:43.622 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:50:43.622 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:50:43.622 INFO:tasks.workunit.client.0.smithi125.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00876548 s, 120 MB/s 2023-10-09T06:50:43.622 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_245068/sparse2 -s 2M 2023-10-09T06:50:43.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:50:43.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:43 smithi125 bash[77299]: cluster 2023-10-09T06:50:42.796289+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:43 smithi125 bash[80002]: cluster 2023-10-09T06:50:42.796289+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e349 with expected crc 2023-10-09T06:50:43.674 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.677 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:50:43.677 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:50:43.682 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:43.682 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_245068/sparse1 2023-10-09T06:50:43.702 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:43.784 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:50:43.785 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:43.785 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:43.790 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:50:43.790 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse1 2023-10-09T06:50:43.790 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-09T06:50:43.807 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:43.836 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:43.845 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 2 MiB 2 2023-10-09T06:50:43.846 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse1 2023-10-09T06:50:43.846 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:50:43.846 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:50:43.846 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:43.846 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:43.896 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.b56d47e9a04e 2023-10-09T06:50:43.897 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.b56d47e9a04e 2023-10-09T06:50:43.897 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse1 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.b56d47e9a04e\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:50:43.898 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:50:43.941 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:43.951 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:43.951 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:43.951 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:43.952 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:43.952 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:43.968 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:43.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:43.999 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:50:43.999 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:50:43.999 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:50:43.999 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout:{ 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "size": 20, 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 600, 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "ops": [ 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45605.0:155849 9.d 9:bf356b8d:::benchmark_data_smithi125_199533_object51318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e263)", 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:50:44.036 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 492.51858093099997, 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0983538209999999, 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45605", 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/1678125772", 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 155849 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.037 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509698+0000", 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.038 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509694+0000", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.6396999999999999e-05 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509710+0000", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6316999999999998e-05 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509747+0000", 2023-10-09T06:50:44.039 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.7050000000000001e-05 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509764+0000", 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3009000000000005e-05 2023-10-09T06:50:44.040 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509837+0000", 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.5705e-05 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:31.509853+0000", 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0938460160000001 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.041 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.042 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.042 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603699+0000", 2023-10-09T06:50:44.043 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013094199999999999 2023-10-09T06:50:44.043 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.043 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.043 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603830+0000", 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00010252299999999999 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 1,4,6", 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.603932+0000", 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002793626 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.044 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.606726+0000", 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042365099999999999 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.045 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607149+0000", 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00042726500000000002 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607577+0000", 2023-10-09T06:50:44.046 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00035003900000000001 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607927+0000", 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9614000000000003e-05 2023-10-09T06:50:44.047 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.607976+0000", 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5104000000000005e-05 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:42:32.608051+0000", 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0983538209999999 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.048 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98239 10.b 10.51fb4b (undecoded) ondisk+write+known_if_redirected e304)", 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.549396+0000", 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 216.47888229500001, 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95985835500000005, 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.049 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98239 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549396+0000", 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.050 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549396+0000", 2023-10-09T06:50:44.051 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.051 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.051 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.051 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.051 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549394+0000", 2023-10-09T06:50:44.052 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0400000000000004e-06 2023-10-09T06:50:44.052 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.052 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.052 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549401+0000", 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.0652999999999999e-05 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549412+0000", 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.066e-06 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.053 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.549418+0000", 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0028335679999999999 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.552252+0000", 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2648e-05 2023-10-09T06:50:44.054 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.552264+0000", 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95698310399999997 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.509247+0000", 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2080000000000003e-06 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.055 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.509255+0000", 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.95985835500000005 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:98279 10.b 10.342c266b (undecoded) ondisk+write+known_if_redirected e304)", 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:07.604670+0000", 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 216.423608328, 2023-10-09T06:50:44.056 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90476240299999999, 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "reached pg", 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 98279 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.057 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604670+0000", 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604670+0000", 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.058 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604668+0000", 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4769999999999997e-06 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.059 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604676+0000", 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4897e-05 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604691+0000", 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8460000000000002e-06 2023-10-09T06:50:44.060 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604698+0000", 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8849000000000004e-05 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.061 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.062 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604767+0000", 2023-10-09T06:50:44.062 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1433e-05 2023-10-09T06:50:44.062 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.062 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:07.604778+0000", 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90464134200000001 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.509419+0000", 2023-10-09T06:50:44.063 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3242e-05 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:08.509433+0000", 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.90476240299999999 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.064 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135297 10.8 10:17115be5:::benchmark_data_smithi125_222801_object37411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.118305+0000", 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.909973186, 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21316121800000001, 2023-10-09T06:50:44.065 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135297 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.066 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118305+0000", 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.067 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118305+0000", 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118304+0000", 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.694e-06 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.068 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118309+0000", 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.7569999999999999e-06 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118319+0000", 2023-10-09T06:50:44.069 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.934e-06 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118326+0000", 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00019933799999999999 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118525+0000", 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00026699299999999998 2023-10-09T06:50:44.070 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118792+0000", 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000167596 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.118960+0000", 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.01340214 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.071 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.132362+0000", 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003059591 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135422+0000", 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19589593999999999 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.072 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.331318+0000", 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4800000000000002e-05 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.331392+0000", 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.4124999999999999e-05 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.073 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.331466+0000", 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21316121800000001 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135304 10.c 10:3fb2b8ef:::benchmark_data_smithi125_222801_object37418:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.074 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.121975+0000", 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.90630358300001, 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21604531399999999, 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135304 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.075 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.121975+0000", 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.121975+0000", 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.076 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.121973+0000", 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1000000000000004e-06 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.121981+0000", 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2403e-05 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.077 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.121994+0000", 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9360000000000002e-06 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122001+0000", 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00044527299999999999 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.078 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122446+0000", 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011389500000000001 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122560+0000", 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012756700000000001 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.079 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122687+0000", 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.008489758 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131177+0000", 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0045677529999999999 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135745+0000", 2023-10-09T06:50:44.080 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.202174937 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.337920+0000", 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9230999999999997e-05 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.081 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.337969+0000", 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1116000000000003e-05 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338020+0000", 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21604531399999999 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.082 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135305 10.5 10:a57cf5c7:::benchmark_data_smithi125_222801_object37419:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.122017+0000", 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.90626166999999, 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21891440000000001, 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.083 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135305 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122017+0000", 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.084 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122017+0000", 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122016+0000", 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.2389999999999998e-06 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.085 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122019+0000", 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.3790000000000003e-06 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122026+0000", 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1459999999999998e-06 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122030+0000", 2023-10-09T06:50:44.086 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.1400999999999999e-05 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122071+0000", 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012940400000000001 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122201+0000", 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014681399999999999 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.087 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122348+0000", 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0057523230000000002 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.088 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.128100+0000", 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010013611 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.138114+0000", 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20270759499999999 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.089 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.340821+0000", 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8032000000000003e-05 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.340879+0000", 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2003999999999999e-05 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.090 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.340931+0000", 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21891440000000001 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135307 10.c 10:382c4322:::benchmark_data_smithi125_222801_object37421:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.122041+0000", 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.906237213, 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21611008800000001, 2023-10-09T06:50:44.091 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135307 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122041+0000", 2023-10-09T06:50:44.092 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122041+0000", 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.093 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122041+0000", 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.3069999999999998e-06 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122044+0000", 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.8530000000000003e-06 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.094 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122051+0000", 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4100000000000001e-06 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.122055+0000", 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0033524290000000001 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.095 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.125408+0000", 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000113581 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.125521+0000", 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013306700000000001 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.096 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.125654+0000", 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0081367890000000002 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.133791+0000", 2023-10-09T06:50:44.097 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.006412287 2023-10-09T06:50:44.098 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.098 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.098 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.098 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.140204+0000", 2023-10-09T06:50:44.098 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.197863179 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338067+0000", 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4412000000000001e-05 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.099 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338111+0000", 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0215e-05 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.100 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338151+0000", 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.21611008800000001 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.101 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135312 10.9 10:92d999d0:::benchmark_data_smithi125_222801_object37426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.131416+0000", 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.89686252999999, 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.206202739, 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.102 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135312 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131416+0000", 2023-10-09T06:50:44.103 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131416+0000", 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.104 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131414+0000", 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8880000000000002e-06 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131420+0000", 2023-10-09T06:50:44.105 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8659999999999995e-06 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131429+0000", 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9439999999999996e-06 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.106 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131434+0000", 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0138000000000001e-05 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131474+0000", 2023-10-09T06:50:44.107 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.2751999999999996e-05 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131557+0000", 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6837999999999998e-05 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.108 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.131644+0000", 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0026525640000000001 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.134296+0000", 2023-10-09T06:50:44.109 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0046128940000000002 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.138909+0000", 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.198610235 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.110 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.337519+0000", 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2516999999999999e-05 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.111 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.337572+0000", 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6792000000000002e-05 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.337619+0000", 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.206202739 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.112 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:135314 10.c 10:3f48ca8b:::benchmark_data_smithi125_222801_object37428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e310)", 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:47:57.135591+0000", 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 166.892687169, 2023-10-09T06:50:44.113 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20267446, 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 135314 2023-10-09T06:50:44.114 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135591+0000", 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.115 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135591+0000", 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135589+0000", 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.9700000000000002e-06 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.116 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135596+0000", 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2645e-05 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135609+0000", 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.4399999999999996e-06 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135614+0000", 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017102 2023-10-09T06:50:44.117 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135785+0000", 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8616000000000007e-05 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135874+0000", 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3014000000000001e-05 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.118 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.135967+0000", 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002049861 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.138017+0000", 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0023652600000000001 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.119 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.140382+0000", 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.19780678700000001 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338189+0000", 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0488999999999998e-05 2023-10-09T06:50:44.120 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338229+0000", 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6461999999999998e-05 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.121 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:47:57.338266+0000", 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.20267446 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.122 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:170484 10.c 10:31cdf482:::benchmark_data_smithi125_222801_object72598:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:48:42.373163+0000", 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 121.655115685, 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14239064400000001, 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.123 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 170484 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.124 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373163+0000", 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.125 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373163+0000", 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373162+0000", 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.5459999999999997e-06 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373169+0000", 2023-10-09T06:50:44.126 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.2130000000000002e-06 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373176+0000", 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.357e-06 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.373180+0000", 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0074809890000000004 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.127 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.380661+0000", 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.5023000000000003e-05 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.380736+0000", 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3399999999999994e-05 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.128 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.380820+0000", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.120312524 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.501132+0000", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7088000000000005e-05 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.501199+0000", 2023-10-09T06:50:44.129 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014252781000000001 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.515452+0000", 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5785000000000002e-05 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.515508+0000", 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5655000000000002e-05 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.130 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:48:42.515553+0000", 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14239064400000001 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187611 10.9 10:97fc12fd:::benchmark_data_smithi125_222801_object89725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.358364+0000", 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.669914435999999, 2023-10-09T06:50:44.131 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.280020149, 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187611 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358364+0000", 2023-10-09T06:50:44.132 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358364+0000", 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358362+0000", 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0919999999999997e-06 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.133 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358368+0000", 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.4420000000000004e-06 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358377+0000", 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2639999999999999e-06 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.134 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358382+0000", 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1983e-05 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358434+0000", 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000103244 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358537+0000", 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00013800600000000001 2023-10-09T06:50:44.135 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358675+0000", 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.012286045000000001 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.370962+0000", 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002040746 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.136 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.373002+0000", 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26528579400000002 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638288+0000", 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5519000000000002e-05 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638344+0000", 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0581999999999999e-05 2023-10-09T06:50:44.137 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638384+0000", 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.280020149 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187612 10.9 10:977bb4ee:::benchmark_data_smithi125_222801_object89726:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.358395+0000", 2023-10-09T06:50:44.138 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.669883089999999, 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.28011321700000003, 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187612 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.139 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358395+0000", 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358395+0000", 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358395+0000", 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9349999999999999e-06 2023-10-09T06:50:44.140 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358398+0000", 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.0970000000000002e-06 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358404+0000", 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7469999999999996e-06 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.141 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.358409+0000", 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0038109979999999999 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.362220+0000", 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000106989 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.142 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.362327+0000", 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000121822 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.362448+0000", 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.010561790999999999 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.373010+0000", 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.001181185 2023-10-09T06:50:44.143 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.374191+0000", 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26423165599999998 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638423+0000", 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7725000000000001e-05 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.144 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638471+0000", 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.7795e-05 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638509+0000", 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.28011321700000003 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.145 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187616 10.9 10:97e4ba06:::benchmark_data_smithi125_222801_object89730:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.361958+0000", 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.666319986000005, 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.27666790800000002, 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187616 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.146 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361958+0000", 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361958+0000", 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.147 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361957+0000", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4007e-05 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361971+0000", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.2578999999999999e-05 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361984+0000", 2023-10-09T06:50:44.148 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.0679999999999999e-06 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.361991+0000", 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0066469629999999997 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.368638+0000", 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1389000000000003e-05 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.149 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.368719+0000", 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.3234999999999998e-05 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.368802+0000", 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050577640000000002 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.150 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.373860+0000", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0050954939999999999 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.378956+0000", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25958774200000001 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638543+0000", 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6718000000000002e-05 2023-10-09T06:50:44.151 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638590+0000", 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.6415000000000003e-05 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.638626+0000", 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.27666790800000002 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.152 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187617 10.b 10:db024ce1:::benchmark_data_smithi125_222801_object89731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.364117+0000", 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.664161172999997, 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.269986638, 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.153 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187617 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364117+0000", 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364117+0000", 2023-10-09T06:50:44.154 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364116+0000", 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3889999999999998e-06 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364120+0000", 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3760000000000002e-06 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.155 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364127+0000", 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.8620000000000002e-06 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364132+0000", 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00063118099999999997 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.156 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364764+0000", 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000105371 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.364869+0000", 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000135272 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.365004+0000", 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0065074859999999998 2023-10-09T06:50:44.157 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.371512+0000", 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022961069999999999 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.373808+0000", 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25982361399999998 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.158 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.633631+0000", 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034547700000000001 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.633977+0000", 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012706600000000001 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.159 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.634104+0000", 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.269986638 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187628 10.5 10:a96d35e2:::benchmark_data_smithi125_222801_object89742:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.380358+0000", 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.647920919000001, 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26152118600000002, 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.160 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187628 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380358+0000", 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.161 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380358+0000", 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380356+0000", 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0300000000000001e-06 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.162 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380361+0000", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.3247999999999999e-05 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380375+0000", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.9810000000000003e-06 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380379+0000", 2023-10-09T06:50:44.163 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.4404999999999999e-05 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380424+0000", 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.3397e-05 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380517+0000", 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.8782000000000004e-05 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.164 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.380616+0000", 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034019979999999998 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384018+0000", 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000129832 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.165 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384148+0000", 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.25751694200000003 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.641665+0000", 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00014010100000000001 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.641805+0000", 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3777999999999997e-05 2023-10-09T06:50:44.166 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.641879+0000", 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.26152118600000002 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:187630 10.8 10:190c3b9b:::benchmark_data_smithi125_222801_object89744:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:07.384451+0000", 2023-10-09T06:50:44.167 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 96.643827131999998, 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.24993193699999999, 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 187630 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.168 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384451+0000", 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384451+0000", 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999981 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384449+0000", 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.6130000000000004e-06 2023-10-09T06:50:44.169 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384458+0000", 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1151999999999999e-05 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384469+0000", 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.8340000000000004e-06 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.170 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384475+0000", 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.2704000000000002e-05 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384527+0000", 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016059200000000001 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.171 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384688+0000", 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00017767099999999999 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.384866+0000", 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0022965419999999999 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.387162+0000", 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00082899000000000004 2023-10-09T06:50:44.172 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.387991+0000", 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.24629357700000001 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.634285+0000", 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5402000000000003e-05 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.173 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.634340+0000", 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.3000999999999997e-05 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:07.634383+0000", 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.24993193699999999 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.174 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:195657 10.5 10:a336377c:::benchmark_data_smithi125_222801_object97771:head [delete] snapc 0=[] ondisk+write+known_if_redirected e325)", 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:22.047851+0000", 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 81.980427745, 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14862234699999999, 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 195657 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.175 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047851+0000", 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047851+0000", 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999995 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.176 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047850+0000", 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.5940000000000001e-06 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047853+0000", 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.5759999999999996e-06 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.177 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047858+0000", 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.6779999999999999e-06 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.047863+0000", 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002283209 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.050146+0000", 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011655 2023-10-09T06:50:44.178 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.050263+0000", 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000137205 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.050400+0000", 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0034010300000000002 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.179 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.053801+0000", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012635750000000001 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.055065+0000", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.141167826 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.196232+0000", 2023-10-09T06:50:44.180 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016003299999999999 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.196392+0000", 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.0638999999999998e-05 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.196473+0000", 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14862234699999999 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.181 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:195666 10.9 10:9d61a535:::benchmark_data_smithi125_222801_object97780:head [delete] snapc 0=[] ondisk+write+known_if_redirected e325)", 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:22.057641+0000", 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 81.970637648999997, 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14267623400000001, 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.182 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 195666 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057641+0000", 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057641+0000", 2023-10-09T06:50:44.183 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999986 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057639+0000", 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.946e-06 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057643+0000", 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.3370000000000002e-06 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.184 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057651+0000", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.0769999999999998e-06 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057656+0000", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.6008999999999999e-05 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057712+0000", 2023-10-09T06:50:44.185 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000109227 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057821+0000", 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000116104 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.057937+0000", 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0029648719999999999 2023-10-09T06:50:44.186 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.060902+0000", 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00011694899999999999 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.061019+0000", 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.13915271300000001 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.187 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.200172+0000", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 7.7675000000000004e-05 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.200249+0000", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.7834000000000001e-05 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.200317+0000", 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.14267623400000001 2023-10-09T06:50:44.188 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.45821.0:195673 10.9 10:960736f2:::benchmark_data_smithi125_222801_object97787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e325)", 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:22.062288+0000", 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 81.965990304000002, 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15498456299999999, 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.189 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.45821", 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4278669412", 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 195673 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062288+0000", 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.190 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062288+0000", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.999999 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062287+0000", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.5170000000000001e-06 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062291+0000", 2023-10-09T06:50:44.191 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 9.0229999999999998e-06 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062301+0000", 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 5.1630000000000004e-06 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062306+0000", 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 6.2038999999999993e-05 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.192 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062368+0000", 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00016357000000000001 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062531+0000", 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.000114014 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:50:44.193 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.062645+0000", 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0025561120000000001 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.065201+0000", 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00012856599999999999 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.065330+0000", 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15185732199999999 2023-10-09T06:50:44.194 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.217187+0000", 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.7240000000000002e-05 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.217235+0000", 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 3.8170000000000002e-05 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.195 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:22.217273+0000", 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.15498456299999999 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "description": "osd_op(client.36135.0:2 11.2 11:424638ea:::image1:head [create] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "initiated_at": "2023-10-09T06:49:59.615834+0000", 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "age": 44.412444669999999, 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46975282400000001, 2023-10-09T06:50:44.196 INFO:teuthology.orchestra.run.smithi155.stdout: "type_data": { 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "client_info": { 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "client": "client.36135", 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "client_addr": "172.21.15.125:0/4152572583", 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "tid": 2 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "events": [ 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "initiated", 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615834+0000", 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0 2023-10-09T06:50:44.197 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "throttled", 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615834+0000", 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4294967295.9999967 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "header_read", 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615831+0000", 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.4566000000000001e-05 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.198 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "all_read", 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615845+0000", 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9805e-05 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "dispatched", 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615875+0000", 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.2050999999999999e-05 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "queued_for_pg", 2023-10-09T06:50:44.199 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615897+0000", 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.8084999999999994e-05 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615985+0000", 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 8.1759999999999993e-06 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op must wait for map", 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.615993+0000", 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.014532394000000001 2023-10-09T06:50:44.200 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.630526+0000", 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 1.1464999999999999e-05 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for peered", 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:49:59.630537+0000", 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.44707577900000001 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.201 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "reached_pg", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.077613+0000", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00034132100000000001 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "started", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.077954+0000", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.00036866600000000002 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.078323+0000", 2023-10-09T06:50:44.202 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.002484521 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.080807+0000", 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.0012907789999999999 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "op_commit", 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.082098+0000", 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.003417649 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.203 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.085516+0000", 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 4.0966e-05 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "commit_sent", 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.085557+0000", 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 2.9853e-05 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: }, 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: { 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "event": "done", 2023-10-09T06:50:44.204 INFO:teuthology.orchestra.run.smithi155.stdout: "time": "2023-10-09T06:50:00.085587+0000", 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: "duration": 0.46975282400000001 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: } 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout: ] 2023-10-09T06:50:44.205 INFO:teuthology.orchestra.run.smithi155.stdout:} 2023-10-09T06:50:44.205 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse1 --format %b 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:50:44.207 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse1.out --format %b 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:44.208 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:50:44.224 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:50:44.230 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:50:44.239 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:44.241 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:50:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:44.300 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 0% complete...failed. 2023-10-09T06:50:44.300 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:50:44.304 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:44.305 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_245068/sparse2 2023-10-09T06:50:44.343 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:44.367 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:44.392 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:50:44.393 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:44.393 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:44.398 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:50:44.398 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grepgrep sparse2 -Ei 2023-10-09T06:50:44.398 INFO:tasks.workunit.client.0.smithi125.stderr: (2 MiB|2048k) 2023-10-09T06:50:44.451 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 2 MiB 2 2023-10-09T06:50:44.452 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse2 2023-10-09T06:50:44.452 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:50:44.452 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:50:44.452 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:44.453 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:44.471 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:44.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[77299]: audit 2023-10-09T06:50:43.473184+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.482 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[77299]: cluster 2023-10-09T06:50:43.501075+0000 mgr.y (mgr.25723) 1391 : cluster 0 pgmap v1540: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 192 KiB/s rd, 18 KiB/s wr, 257 op/s 2023-10-09T06:50:44.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[77299]: audit 2023-10-09T06:50:43.580957+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.483 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[77299]: cluster 2023-10-09T06:50:44.175488+0000 mon.a (mon.0) 1728 : cluster 0 osdmap e350: 8 total, 8 up, 4 in 2023-10-09T06:50:44.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[80002]: audit 2023-10-09T06:50:43.473184+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.483 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[80002]: cluster 2023-10-09T06:50:43.501075+0000 mgr.y (mgr.25723) 1391 : cluster 0 pgmap v1540: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 192 KiB/s rd, 18 KiB/s wr, 257 op/s 2023-10-09T06:50:44.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[80002]: audit 2023-10-09T06:50:43.580957+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.484 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:44 smithi125 bash[80002]: cluster 2023-10-09T06:50:44.175488+0000 mon.a (mon.0) 1728 : cluster 0 osdmap e350: 8 total, 8 up, 4 in 2023-10-09T06:50:44.499 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:44.501 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.b5a0fdceba00 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.b5a0fdceba00 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.b5a0fdceba00\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ + sort -u 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:get_image_data_pool sparse2 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:50:44.503 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:50:44.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:44.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:44.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:44.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:44.558 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:44.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:44.607 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:50:44.607 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2023-10-09T06:50:44.607 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T06:50:44.608 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:44.636 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:44 smithi155 bash[59495]: audit 2023-10-09T06:50:43.473184+0000 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:44 smithi155 bash[59495]: cluster 2023-10-09T06:50:43.501075+0000 mgr.y (mgr.25723) 1391 : cluster 0 pgmap v1540: 73 pgs: 73 active+clean; 4.8 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 192 KiB/s rd, 18 KiB/s wr, 257 op/s 2023-10-09T06:50:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:44 smithi155 bash[59495]: audit 2023-10-09T06:50:43.580957+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.125:0/1149011155' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:50:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:44 smithi155 bash[59495]: cluster 2023-10-09T06:50:44.175488+0000 mon.a (mon.0) 1728 : cluster 0 osdmap e350: 8 total, 8 up, 4 in 2023-10-09T06:50:44.655 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:44.661 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:44.661 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:44.665 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse2 --format %b 2023-10-09T06:50:44.666 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:50:44.666 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse2.out --format %b 2023-10-09T06:50:44.667 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:50:44.667 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:44.668 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:44.668 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:50:44.668 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:44.669 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:50:44.741 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:44.770 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:44.821 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:50:44.826 INFO:tasks.workunit.client.0.smithi125.stderr:+ truncate /tmp/rbd_import_export_245068/sparse1 -s 10M 2023-10-09T06:50:44.827 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T06:50:44.873 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:44.907 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:44.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:44.976 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:44.977 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:44.978 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:44.982 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:50:44.982 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grep sparse1 2023-10-09T06:50:44.982 INFO:tasks.workunit.client.0.smithi125.stderr:grep -Ei (10 MiB|10240k) 2023-10-09T06:50:45.010 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:45.035 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:50:45.040 INFO:tasks.workunit.client.0.smithi125.stdout:sparse1 10 MiB 2 2023-10-09T06:50:45.040 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse1 2023-10-09T06:50:45.041 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:45.041 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:50:45.042 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:50:45.042 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:45.043 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:45.096 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8f22873c5347 2023-10-09T06:50:45.098 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.8f22873c5347 2023-10-09T06:50:45.098 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse1 2023-10-09T06:50:45.098 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse1 2023-10-09T06:50:45.098 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.8f22873c5347\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:45.099 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:45.099 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse1 2023-10-09T06:50:45.099 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grep data_pool: 2023-10-09T06:50:45.099 INFO:tasks.workunit.client.0.smithi125.stderr:awk -F: { print $NF } 2023-10-09T06:50:45.146 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.152 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:45.153 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:45.153 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:45.153 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:45.153 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:45.176 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:45.197 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:50:45.197 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:50:45.197 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:50:45.197 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:45.250 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:45.254 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:45.254 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/sparse1 /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:45.265 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse1 --format %b 2023-10-09T06:50:45.266 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=2048 2023-10-09T06:50:45.267 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse1.out --format %b 2023-10-09T06:50:45.268 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=2048 2023-10-09T06:50:45.268 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:45.269 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:45.269 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:50:45.269 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse1.out 2023-10-09T06:50:45.271 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse1 2023-10-09T06:50:45.280 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.311 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:45.357 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:50:45.415 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.428 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:50:45.433 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_245068/sparse2 oflag=append conv=notrunc 2023-10-09T06:50:45.445 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:45.446 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:50:45.446 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:50:45.446 INFO:tasks.workunit.client.0.smithi125.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0113486 s, 185 MB/s 2023-10-09T06:50:45.446 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T06:50:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:45.549 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:45.568 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:45.569 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:45.569 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:45.573 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd ls -l 2023-10-09T06:50:45.573 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep sparse2 2023-10-09T06:50:45.573 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-09T06:50:45.574 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:45.622 INFO:tasks.workunit.client.0.smithi125.stdout:sparse2 4 MiB 2 2023-10-09T06:50:45.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse2 2023-10-09T06:50:45.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:50:45.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:50:45.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:45.623 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:45.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[77299]: cluster 2023-10-09T06:50:44.264562+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[77299]: cluster 2023-10-09T06:50:44.276808+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[77299]: cluster 2023-10-09T06:50:44.550490+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[80002]: cluster 2023-10-09T06:50:44.264562+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[80002]: cluster 2023-10-09T06:50:44.276808+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:45 smithi125 bash[80002]: cluster 2023-10-09T06:50:44.550490+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:45 smithi155 bash[59495]: cluster 2023-10-09T06:50:44.264562+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:45 smithi155 bash[59495]: cluster 2023-10-09T06:50:44.276808+0000 osd.4 (osd.4) 295 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:45 smithi155 bash[59495]: cluster 2023-10-09T06:50:44.550490+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e350 with expected crc 2023-10-09T06:50:45.677 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8f4f164e5335 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.8f4f164e5335 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ + sed -eget_image_data_pool s/rbd_data.8f4f164e5335\.// -e sparse2 s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse2 2023-10-09T06:50:45.679 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:45.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse2 2023-10-09T06:50:45.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:50:45.680 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:50:45.711 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:45.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:45.735 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:45.735 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:45.735 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:45.735 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:45.735 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:2 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:3 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2 3 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-09T06:50:45.786 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:45.815 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:45.844 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:45.847 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T06:50:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:45.852 INFO:tasks.workunit.client.0.smithi125.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_245068/sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:45.852 INFO:tasks.workunit.client.0.smithi125.stderr:+ cmp -l /tmp/rbd_import_export_245068/sparse2 /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:45.856 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse2 --format %b 2023-10-09T06:50:45.857 INFO:tasks.workunit.client.0.smithi125.stderr:+ origsize=6144 2023-10-09T06:50:45.858 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse2.out --format %b 2023-10-09T06:50:45.859 INFO:tasks.workunit.client.0.smithi125.stderr:+ exportsize=6144 2023-10-09T06:50:45.859 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:45.859 INFO:tasks.workunit.client.0.smithi125.stderr:+ difference=0 2023-10-09T06:50:45.859 INFO:tasks.workunit.client.0.smithi125.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:50:45.859 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse2.out 2023-10-09T06:50:45.860 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse2 2023-10-09T06:50:45.871 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:45.949 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:45.976 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:45.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:45.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:46.015 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T06:50:46.020 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-09T06:50:46.021 INFO:tasks.workunit.client.0.smithi125.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T06:50:46.021 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_245068/sparse1 sparse 2023-10-09T06:50:46.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.102 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:46.112 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T06:50:46.113 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:46.113 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:50:46.117 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:50:46.117 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:46.117 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.117 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:46.117 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:46.164 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8f7cfc400f24 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.8f7cfc400f24 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.8f7cfc400f24\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.166 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grep data_pool: 2023-10-09T06:50:46.167 INFO:tasks.workunit.client.0.smithi125.stderr:awk -F: { print $NF } 2023-10-09T06:50:46.205 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:46.212 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:46.212 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:46.212 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:46.213 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:46.213 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:46.234 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:46.252 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=1 2023-10-09T06:50:46.252 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 1 2023-10-09T06:50:46.252 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-09T06:50:46.252 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:50:46.339 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:46.366 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:46.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:46.396 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:50:46.400 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo zeros import through stdin to sparse image 2023-10-09T06:50:46.400 INFO:tasks.workunit.client.0.smithi125.stdout:zeros import through stdin to sparse image 2023-10-09T06:50:46.401 INFO:tasks.workunit.client.0.smithi125.stderr:+ + dd if=/dev/zero bs=1M count=4 2023-10-09T06:50:46.401 INFO:tasks.workunit.client.0.smithi125.stderr:rbd import --new-format - sparse 2023-10-09T06:50:46.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.474 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records in 2023-10-09T06:50:46.474 INFO:tasks.workunit.client.0.smithi125.stderr:4+0 records out 2023-10-09T06:50:46.474 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0722975 s, 58.0 MB/s 2023-10-09T06:50:46.493 INFO:tasks.workunit.client.0.smithi125.stderr: Importing image: 100% complete...done. 2023-10-09T06:50:46.494 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:46.500 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.501 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:50:46.501 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:46.501 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.502 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:46.502 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:46.552 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8f9d22267fd9 2023-10-09T06:50:46.553 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.8f9d22267fd9 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.8f9d22267fd9\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:50:46.554 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:50:46.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:46.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:46.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:46.602 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:46.603 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:46.604 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:46.627 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:50:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:50:46.645 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects= 2023-10-09T06:50:46.645 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 2023-10-09T06:50:46.645 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o = ] 2023-10-09T06:50:46.645 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:50:46.731 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:46.743 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:46 smithi125 bash[77299]: cluster 2023-10-09T06:50:45.502032+0000 mgr.y (mgr.25723) 1392 : cluster 0 pgmap v1542: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 546 KiB/s rd, 396 KiB/s wr, 219 op/s 2023-10-09T06:50:46.744 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:46 smithi125 bash[80002]: cluster 2023-10-09T06:50:45.502032+0000 mgr.y (mgr.25723) 1392 : cluster 0 pgmap v1542: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 546 KiB/s rd, 396 KiB/s wr, 219 op/s 2023-10-09T06:50:46.758 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:46.791 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:46.795 INFO:tasks.workunit.client.0.smithi125.stdout:zeros export to sparse file 2023-10-09T06:50:46.796 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo zeros export to sparse file 2023-10-09T06:50:46.796 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T06:50:46.852 INFO:tasks.workunit.client.0.smithi125.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:50:46.856 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.856 INFO:tasks.workunit.client.0.smithi125.stderr:+ + grepawk block_name_prefix {print $NF;} 2023-10-09T06:50:46.856 INFO:tasks.workunit.client.0.smithi125.stderr: 2023-10-09T06:50:46.863 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.891 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:50:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:46 smithi155 bash[59495]: cluster 2023-10-09T06:50:45.502032+0000 mgr.y (mgr.25723) 1392 : cluster 0 pgmap v1542: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 546 KiB/s rd, 396 KiB/s wr, 219 op/s 2023-10-09T06:50:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:46.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:50:46.911 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8fbe1ff3e840 2023-10-09T06:50:46.912 INFO:tasks.workunit.client.0.smithi125.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T06:50:46.912 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:50:46.912 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:46.912 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:46.912 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep+ awk -F: { print $NF } 2023-10-09T06:50:46.913 INFO:tasks.workunit.client.0.smithi125.stderr: data_pool: 2023-10-09T06:50:46.962 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:46.962 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:46.962 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:46.962 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:46.963 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados -p rbd put rbd_data.8fbe1ff3e840.000000000000 - 2023-10-09T06:50:46.996 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:47.002 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records in 2023-10-09T06:50:47.002 INFO:tasks.workunit.client.0.smithi125.stderr:1+0 records out 2023-10-09T06:50:47.002 INFO:tasks.workunit.client.0.smithi125.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0913958 s, 45.9 MB/s 2023-10-09T06:50:47.028 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:50:47.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:50:47.032 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects sparse 2023-10-09T06:50:47.032 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:47.033 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:47.033 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk {print $NF;} 2023-10-09T06:50:47.034 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep block_name_prefix 2023-10-09T06:50:47.086 INFO:tasks.workunit.client.0.smithi125.stderr:+ prefix=rbd_data.8fbe1ff3e840 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep rbd_data.8fbe1ff3e840 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ get_image_data_pool sparse 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ image=sparse 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ sed -e s/rbd_data.8fbe1ff3e840\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ sort -u 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd info sparse 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ grep data_pool: 2023-10-09T06:50:47.088 INFO:tasks.workunit.client.0.smithi125.stderr:+ awk -F: { print $NF } 2023-10-09T06:50:47.131 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:47.136 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool= 2023-10-09T06:50:47.136 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ -z ] 2023-10-09T06:50:47.137 INFO:tasks.workunit.client.0.smithi125.stderr:+ data_pool=rbd 2023-10-09T06:50:47.137 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo rbd 2023-10-09T06:50:47.137 INFO:tasks.workunit.client.0.smithi125.stderr:+ rados ls -p rbd 2023-10-09T06:50:47.160 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:47.178 INFO:tasks.workunit.client.0.smithi125.stderr:+ objects=0 2023-10-09T06:50:47.178 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo 0 2023-10-09T06:50:47.178 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-09T06:50:47.179 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse 2023-10-09T06:50:47.179 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_245068/sparse': No such file or directory 2023-10-09T06:50:47.179 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:47.180 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd export sparse /tmp/rbd_import_export_245068/sparse 2023-10-09T06:50:47.234 INFO:tasks.workunit.client.0.smithi125.stderr: Exporting image: 100% complete...done. 2023-10-09T06:50:47.239 INFO:tasks.workunit.client.0.smithi125.stderr:+ stat /tmp/rbd_import_export_245068/sparse --format=%b 2023-10-09T06:50:47.241 INFO:tasks.workunit.client.0.smithi125.stderr:+ [ 0 = 0 ] 2023-10-09T06:50:47.241 INFO:tasks.workunit.client.0.smithi125.stderr:+ rbd rm sparse 2023-10-09T06:50:47.266 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:47.292 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:47.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:47.394 INFO:tasks.workunit.client.0.smithi125.stderr: Removing image: 100% complete...done. 2023-10-09T06:50:47.395 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:47.399 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm /tmp/rbd_import_export_245068/sparse /tmp/rbd_import_export_245068/sparse1 /tmp/rbd_import_export_245068/sparse2 /tmp/rbd_import_export_245068/sparse3 2023-10-09T06:50:47.402 INFO:tasks.workunit.client.0.smithi125.stderr:rm: cannot remove '/tmp/rbd_import_export_245068/sparse3': No such file or directory 2023-10-09T06:50:47.403 INFO:tasks.workunit.client.0.smithi125.stderr:+ true 2023-10-09T06:50:47.403 INFO:tasks.workunit.client.0.smithi125.stdout:OK 2023-10-09T06:50:47.404 INFO:tasks.workunit.client.0.smithi125.stderr:+ echo OK 2023-10-09T06:50:47.404 INFO:tasks.workunit.client.0.smithi125.stderr:+ rm -rf /tmp/rbd_import_export_245068 2023-10-09T06:50:47.404 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:50:47.405 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:50:47.415 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T06:50:47.415 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:50:47.425 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:50:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:50:47.529 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:47.556 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:50:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:50:47.660 DEBUG:teuthology.orchestra.run.smithi155:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:47.674 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:47 smithi155 bash[59495]: cluster 2023-10-09T06:50:47.502665+0000 mgr.y (mgr.25723) 1393 : cluster 0 pgmap v1543: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 461 KiB/s rd, 334 KiB/s wr, 184 op/s 2023-10-09T06:50:47.691 INFO:teuthology.orchestra.run.smithi155.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:50:47.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi155 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:50:47.720 DEBUG:teuthology.parallel:result is None 2023-10-09T06:50:47.721 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:47.731 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:47.731 DEBUG:teuthology.orchestra.run.smithi125:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:47.779 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:50:47.779 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:50:47.779 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T06:50:47.779 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:50:47.780 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-09T06:50:47.780 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:50:47.780 DEBUG:teuthology.orchestra.run.smithi125:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:47.794 DEBUG:teuthology.orchestra.run.smithi125:> sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.832 INFO:teuthology.orchestra.run.smithi125.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:50:47.832 DEBUG:teuthology.orchestra.run.smithi125:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:47.836 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:47 smithi125 bash[77299]: cluster 2023-10-09T06:50:47.502665+0000 mgr.y (mgr.25723) 1393 : cluster 0 pgmap v1543: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 461 KiB/s rd, 334 KiB/s wr, 184 op/s 2023-10-09T06:50:47.836 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:47 smithi125 bash[80002]: cluster 2023-10-09T06:50:47.502665+0000 mgr.y (mgr.25723) 1393 : cluster 0 pgmap v1543: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 461 KiB/s rd, 334 KiB/s wr, 184 op/s 2023-10-09T06:50:47.842 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:50:47.842 DEBUG:teuthology.orchestra.run.smithi125:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:50:47.853 INFO:teuthology.orchestra.run.smithi125.stderr:Error response from daemon: Cannot kill container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:50:47.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:50:47.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi125 with status 1: 'sudo docker kill -s 1 ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:50:47.860 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T06:50:49.288 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:49.289 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:49 smithi155 bash[59495]: audit 2023-10-09T06:50:48.620537+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:50:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:49 smithi155 bash[59495]: audit 2023-10-09T06:50:48.622215+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:49 smithi155 bash[59495]: cluster 2023-10-09T06:50:49.177731+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e351: 8 total, 8 up, 4 in 2023-10-09T06:50:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:49 smithi155 bash[59495]: cluster 2023-10-09T06:50:49.503253+0000 mgr.y (mgr.25723) 1394 : cluster 0 pgmap v1545: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 398 KiB/s rd, 288 KiB/s wr, 159 op/s 2023-10-09T06:50:49.938 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:50:50.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[77299]: audit 2023-10-09T06:50:48.620537+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[77299]: audit 2023-10-09T06:50:48.622215+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[77299]: cluster 2023-10-09T06:50:49.177731+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e351: 8 total, 8 up, 4 in 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[77299]: cluster 2023-10-09T06:50:49.503253+0000 mgr.y (mgr.25723) 1394 : cluster 0 pgmap v1545: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 398 KiB/s rd, 288 KiB/s wr, 159 op/s 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[80002]: audit 2023-10-09T06:50:48.620537+0000 mon.a (mon.0) 1729 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[80002]: audit 2023-10-09T06:50:48.622215+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[80002]: cluster 2023-10-09T06:50:49.177731+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e351: 8 total, 8 up, 4 in 2023-10-09T06:50:50.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:49 smithi125 bash[80002]: cluster 2023-10-09T06:50:49.503253+0000 mgr.y (mgr.25723) 1394 : cluster 0 pgmap v1545: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 398 KiB/s rd, 288 KiB/s wr, 159 op/s 2023-10-09T06:50:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:50 smithi125 bash[77299]: cluster 2023-10-09T06:50:49.175451+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:50 smithi125 bash[77299]: cluster 2023-10-09T06:50:49.495234+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:50.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:50 smithi125 bash[80002]: cluster 2023-10-09T06:50:49.175451+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:50.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:50 smithi125 bash[80002]: cluster 2023-10-09T06:50:49.495234+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:50 smithi155 bash[59495]: cluster 2023-10-09T06:50:49.175451+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:50 smithi155 bash[59495]: cluster 2023-10-09T06:50:49.495234+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:51.488 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:50:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:50:51] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:50:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:51 smithi125 bash[77299]: cluster 2023-10-09T06:50:50.049088+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:51.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:51 smithi125 bash[77299]: cluster 2023-10-09T06:50:51.503901+0000 mgr.y (mgr.25723) 1395 : cluster 0 pgmap v1546: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-10-09T06:50:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:51 smithi125 bash[80002]: cluster 2023-10-09T06:50:50.049088+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:51 smithi125 bash[80002]: cluster 2023-10-09T06:50:51.503901+0000 mgr.y (mgr.25723) 1395 : cluster 0 pgmap v1546: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-10-09T06:50:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:51 smithi155 bash[59495]: cluster 2023-10-09T06:50:50.049088+0000 osd.4 (osd.4) 296 : cluster 3 failed to encode map e351 with expected crc 2023-10-09T06:50:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:51 smithi155 bash[59495]: cluster 2023-10-09T06:50:51.503901+0000 mgr.y (mgr.25723) 1395 : cluster 0 pgmap v1546: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-10-09T06:50:51.933 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:50:51.935 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:50:52.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:52 smithi125 bash[80002]: audit 2023-10-09T06:50:51.920169+0000 mgr.y (mgr.25723) 1396 : audit 0 from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:52.885 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:52 smithi125 bash[77299]: audit 2023-10-09T06:50:51.920169+0000 mgr.y (mgr.25723) 1396 : audit 0 from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:52.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:52 smithi155 bash[59495]: audit 2023-10-09T06:50:51.920169+0000 mgr.y (mgr.25723) 1396 : audit 0 from='client.36858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:50:53.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:53 smithi125 bash[80002]: cluster 2023-10-09T06:50:53.504580+0000 mgr.y (mgr.25723) 1397 : cluster 0 pgmap v1547: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 875 KiB/s rd, 1.2 MiB/s wr, 290 op/s 2023-10-09T06:50:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:53 smithi125 bash[77299]: cluster 2023-10-09T06:50:53.504580+0000 mgr.y (mgr.25723) 1397 : cluster 0 pgmap v1547: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 875 KiB/s rd, 1.2 MiB/s wr, 290 op/s 2023-10-09T06:50:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:53 smithi155 bash[59495]: cluster 2023-10-09T06:50:53.504580+0000 mgr.y (mgr.25723) 1397 : cluster 0 pgmap v1547: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 875 KiB/s rd, 1.2 MiB/s wr, 290 op/s 2023-10-09T06:50:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:54 smithi125 bash[80002]: audit 2023-10-09T06:50:53.701239+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:50:55.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:54 smithi125 bash[77299]: audit 2023-10-09T06:50:53.701239+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:50:55.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:54 smithi155 bash[59495]: audit 2023-10-09T06:50:53.701239+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:50:55.255 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1546,"stamp":"2023-10-09T06:50:51.503492+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12221,"num_read_kb":25910,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":356,"num_bytes_recovered":7768541,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3680852,"kb_used_data":39204,"kb_used_omap":8,"kb_used_meta":3641591,"kb_avail":371299756,"statfs":{"total":383980142592,"available":380210950144,"internally_reserved":0,"allocated":40144896,"data_stored":37018887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":3728989440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3968852,"num_objects":-9,"num_object_clones":0,"num_object_copies":-27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9,"num_whiteouts":0,"num_read":2374,"num_read_kb":8168,"num_write":339,"num_write_kb":11386,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002750"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":350,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280640+0000","last_change":"2023-10-09T06:45:05.136158+0000","last_active":"2023-10-09T06:50:44.280640+0000","last_peered":"2023-10-09T06:50:44.280640+0000","last_clean":"2023-10-09T06:50:44.280640+0000","last_became_active":"2023-10-09T06:45:05.135763+0000","last_became_peered":"2023-10-09T06:45:05.135763+0000","last_unstale":"2023-10-09T06:50:44.280640+0000","last_undegraded":"2023-10-09T06:50:44.280640+0000","last_fullsized":"2023-10-09T06:50:44.280640+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":331,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281479+0000","last_change":"2023-10-09T06:45:33.529264+0000","last_active":"2023-10-09T06:50:44.281479+0000","last_peered":"2023-10-09T06:50:44.281479+0000","last_clean":"2023-10-09T06:50:44.281479+0000","last_became_active":"2023-10-09T06:45:33.528935+0000","last_became_peered":"2023-10-09T06:45:33.528935+0000","last_unstale":"2023-10-09T06:50:44.281479+0000","last_undegraded":"2023-10-09T06:50:44.281479+0000","last_fullsized":"2023-10-09T06:50:44.281479+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280138+0000","last_change":"2023-10-09T06:49:20.588657+0000","last_active":"2023-10-09T06:50:44.280138+0000","last_peered":"2023-10-09T06:50:44.280138+0000","last_clean":"2023-10-09T06:50:44.280138+0000","last_became_active":"2023-10-09T06:49:20.588048+0000","last_became_peered":"2023-10-09T06:49:20.588048+0000","last_unstale":"2023-10-09T06:50:44.280138+0000","last_undegraded":"2023-10-09T06:50:44.280138+0000","last_fullsized":"2023-10-09T06:50:44.280138+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":217,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270927+0000","last_change":"2023-10-09T06:45:33.542604+0000","last_active":"2023-10-09T06:50:44.270927+0000","last_peered":"2023-10-09T06:50:44.270927+0000","last_clean":"2023-10-09T06:50:44.270927+0000","last_became_active":"2023-10-09T06:45:33.541873+0000","last_became_peered":"2023-10-09T06:45:33.541873+0000","last_unstale":"2023-10-09T06:50:44.270927+0000","last_undegraded":"2023-10-09T06:50:44.270927+0000","last_fullsized":"2023-10-09T06:50:44.270927+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":379,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.268913+0000","last_change":"2023-10-09T06:48:57.562856+0000","last_active":"2023-10-09T06:50:44.268913+0000","last_peered":"2023-10-09T06:50:44.268913+0000","last_clean":"2023-10-09T06:50:44.268913+0000","last_became_active":"2023-10-09T06:48:57.562492+0000","last_became_peered":"2023-10-09T06:48:57.562492+0000","last_unstale":"2023-10-09T06:50:44.268913+0000","last_undegraded":"2023-10-09T06:50:44.268913+0000","last_fullsized":"2023-10-09T06:50:44.268913+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":115,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.290848+0000","last_change":"2023-10-09T06:45:33.532461+0000","last_active":"2023-10-09T06:50:44.290848+0000","last_peered":"2023-10-09T06:50:44.290848+0000","last_clean":"2023-10-09T06:50:44.290848+0000","last_became_active":"2023-10-09T06:45:33.532017+0000","last_became_peered":"2023-10-09T06:45:33.532017+0000","last_unstale":"2023-10-09T06:50:44.290848+0000","last_undegraded":"2023-10-09T06:50:44.290848+0000","last_fullsized":"2023-10-09T06:50:44.290848+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":315,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.290916+0000","last_change":"2023-10-09T06:45:33.522075+0000","last_active":"2023-10-09T06:50:44.290916+0000","last_peered":"2023-10-09T06:50:44.290916+0000","last_clean":"2023-10-09T06:50:44.290916+0000","last_became_active":"2023-10-09T06:45:33.521697+0000","last_became_peered":"2023-10-09T06:45:33.521697+0000","last_unstale":"2023-10-09T06:50:44.290916+0000","last_undegraded":"2023-10-09T06:50:44.290916+0000","last_fullsized":"2023-10-09T06:50:44.290916+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":362,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.289476+0000","last_change":"2023-10-09T06:45:33.532746+0000","last_active":"2023-10-09T06:50:44.289476+0000","last_peered":"2023-10-09T06:50:44.289476+0000","last_clean":"2023-10-09T06:50:44.289476+0000","last_became_active":"2023-10-09T06:45:33.532241+0000","last_became_peered":"2023-10-09T06:45:33.532241+0000","last_unstale":"2023-10-09T06:50:44.289476+0000","last_undegraded":"2023-10-09T06:50:44.289476+0000","last_fullsized":"2023-10-09T06:50:44.289476+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":267,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270907+0000","last_change":"2023-10-09T06:49:20.583506+0000","last_active":"2023-10-09T06:50:44.270907+0000","last_peered":"2023-10-09T06:50:44.270907+0000","last_clean":"2023-10-09T06:50:44.270907+0000","last_became_active":"2023-10-09T06:49:20.583097+0000","last_became_peered":"2023-10-09T06:49:20.583097+0000","last_unstale":"2023-10-09T06:50:44.270907+0000","last_undegraded":"2023-10-09T06:50:44.270907+0000","last_fullsized":"2023-10-09T06:50:44.270907+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":348,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.268786+0000","last_change":"2023-10-09T06:45:33.529016+0000","last_active":"2023-10-09T06:50:44.268786+0000","last_peered":"2023-10-09T06:50:44.268786+0000","last_clean":"2023-10-09T06:50:44.268786+0000","last_became_active":"2023-10-09T06:45:33.528659+0000","last_became_peered":"2023-10-09T06:45:33.528659+0000","last_unstale":"2023-10-09T06:50:44.268786+0000","last_undegraded":"2023-10-09T06:50:44.268786+0000","last_fullsized":"2023-10-09T06:50:44.268786+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":264,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270514+0000","last_change":"2023-10-09T06:45:33.810240+0000","last_active":"2023-10-09T06:50:44.270514+0000","last_peered":"2023-10-09T06:50:44.270514+0000","last_clean":"2023-10-09T06:50:44.270514+0000","last_became_active":"2023-10-09T06:45:33.538067+0000","last_became_peered":"2023-10-09T06:45:33.538067+0000","last_unstale":"2023-10-09T06:50:44.270514+0000","last_undegraded":"2023-10-09T06:50:44.270514+0000","last_fullsized":"2023-10-09T06:50:44.270514+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":106,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.290105+0000","last_change":"2023-10-09T06:45:05.139216+0000","last_active":"2023-10-09T06:50:44.290105+0000","last_peered":"2023-10-09T06:50:44.290105+0000","last_clean":"2023-10-09T06:50:44.290105+0000","last_became_active":"2023-10-09T06:45:05.138786+0000","last_became_peered":"2023-10-09T06:45:05.138786+0000","last_unstale":"2023-10-09T06:50:44.290105+0000","last_undegraded":"2023-10-09T06:50:44.290105+0000","last_fullsized":"2023-10-09T06:50:44.290105+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":437,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280730+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:50:44.280730+0000","last_peered":"2023-10-09T06:50:44.280730+0000","last_clean":"2023-10-09T06:50:44.280730+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:50:44.280730+0000","last_undegraded":"2023-10-09T06:50:44.280730+0000","last_fullsized":"2023-10-09T06:50:44.280730+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":369,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280815+0000","last_change":"2023-10-09T06:45:05.135916+0000","last_active":"2023-10-09T06:50:44.280815+0000","last_peered":"2023-10-09T06:50:44.280815+0000","last_clean":"2023-10-09T06:50:44.280815+0000","last_became_active":"2023-10-09T06:45:05.135486+0000","last_became_peered":"2023-10-09T06:45:05.135486+0000","last_unstale":"2023-10-09T06:50:44.280815+0000","last_undegraded":"2023-10-09T06:50:44.280815+0000","last_fullsized":"2023-10-09T06:50:44.280815+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":264,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269106+0000","last_change":"2023-10-09T06:45:05.231838+0000","last_active":"2023-10-09T06:50:44.269106+0000","last_peered":"2023-10-09T06:50:44.269106+0000","last_clean":"2023-10-09T06:50:44.269106+0000","last_became_active":"2023-10-09T06:45:05.139440+0000","last_became_peered":"2023-10-09T06:45:05.139440+0000","last_unstale":"2023-10-09T06:50:44.269106+0000","last_undegraded":"2023-10-09T06:50:44.269106+0000","last_fullsized":"2023-10-09T06:50:44.269106+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":363,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269196+0000","last_change":"2023-10-09T06:45:33.533944+0000","last_active":"2023-10-09T06:50:44.269196+0000","last_peered":"2023-10-09T06:50:44.269196+0000","last_clean":"2023-10-09T06:50:44.269196+0000","last_became_active":"2023-10-09T06:45:33.533570+0000","last_became_peered":"2023-10-09T06:45:33.533570+0000","last_unstale":"2023-10-09T06:50:44.269196+0000","last_undegraded":"2023-10-09T06:50:44.269196+0000","last_fullsized":"2023-10-09T06:50:44.269196+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":208,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269306+0000","last_change":"2023-10-09T06:45:33.542558+0000","last_active":"2023-10-09T06:50:44.269306+0000","last_peered":"2023-10-09T06:50:44.269306+0000","last_clean":"2023-10-09T06:50:44.269306+0000","last_became_active":"2023-10-09T06:45:33.542133+0000","last_became_peered":"2023-10-09T06:45:33.542133+0000","last_unstale":"2023-10-09T06:50:44.269306+0000","last_undegraded":"2023-10-09T06:50:44.269306+0000","last_fullsized":"2023-10-09T06:50:44.269306+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":248,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270395+0000","last_change":"2023-10-09T06:45:33.533983+0000","last_active":"2023-10-09T06:50:44.270395+0000","last_peered":"2023-10-09T06:50:44.270395+0000","last_clean":"2023-10-09T06:50:44.270395+0000","last_became_active":"2023-10-09T06:45:33.533641+0000","last_became_peered":"2023-10-09T06:45:33.533641+0000","last_unstale":"2023-10-09T06:50:44.270395+0000","last_undegraded":"2023-10-09T06:50:44.270395+0000","last_fullsized":"2023-10-09T06:50:44.270395+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":297,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.268775+0000","last_change":"2023-10-09T06:45:05.141190+0000","last_active":"2023-10-09T06:50:44.268775+0000","last_peered":"2023-10-09T06:50:44.268775+0000","last_clean":"2023-10-09T06:50:44.268775+0000","last_became_active":"2023-10-09T06:45:05.140792+0000","last_became_peered":"2023-10-09T06:45:05.140792+0000","last_unstale":"2023-10-09T06:50:44.268775+0000","last_undegraded":"2023-10-09T06:50:44.268775+0000","last_fullsized":"2023-10-09T06:50:44.268775+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":148,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270754+0000","last_change":"2023-10-09T06:45:33.542855+0000","last_active":"2023-10-09T06:50:44.270754+0000","last_peered":"2023-10-09T06:50:44.270754+0000","last_clean":"2023-10-09T06:50:44.270754+0000","last_became_active":"2023-10-09T06:45:33.542281+0000","last_became_peered":"2023-10-09T06:45:33.542281+0000","last_unstale":"2023-10-09T06:50:44.270754+0000","last_undegraded":"2023-10-09T06:50:44.270754+0000","last_fullsized":"2023-10-09T06:50:44.270754+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":146,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.289654+0000","last_change":"2023-10-09T06:48:58.571870+0000","last_active":"2023-10-09T06:50:44.289654+0000","last_peered":"2023-10-09T06:50:44.289654+0000","last_clean":"2023-10-09T06:50:44.289654+0000","last_became_active":"2023-10-09T06:48:58.571360+0000","last_became_peered":"2023-10-09T06:48:58.571360+0000","last_unstale":"2023-10-09T06:50:44.289654+0000","last_undegraded":"2023-10-09T06:50:44.289654+0000","last_fullsized":"2023-10-09T06:50:44.289654+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":385,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270809+0000","last_change":"2023-10-09T06:48:57.566980+0000","last_active":"2023-10-09T06:50:44.270809+0000","last_peered":"2023-10-09T06:50:44.270809+0000","last_clean":"2023-10-09T06:50:44.270809+0000","last_became_active":"2023-10-09T06:48:57.566645+0000","last_became_peered":"2023-10-09T06:48:57.566645+0000","last_unstale":"2023-10-09T06:50:44.270809+0000","last_undegraded":"2023-10-09T06:50:44.270809+0000","last_fullsized":"2023-10-09T06:50:44.270809+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":247,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270719+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:50:44.270719+0000","last_peered":"2023-10-09T06:50:44.270719+0000","last_clean":"2023-10-09T06:50:44.270719+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:50:44.270719+0000","last_undegraded":"2023-10-09T06:50:44.270719+0000","last_fullsized":"2023-10-09T06:50:44.270719+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":294,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269747+0000","last_change":"2023-10-09T06:42:32.485962+0000","last_active":"2023-10-09T06:50:44.269747+0000","last_peered":"2023-10-09T06:50:44.269747+0000","last_clean":"2023-10-09T06:50:44.269747+0000","last_became_active":"2023-10-09T06:42:32.485115+0000","last_became_peered":"2023-10-09T06:42:32.485115+0000","last_unstale":"2023-10-09T06:50:44.269747+0000","last_undegraded":"2023-10-09T06:50:44.269747+0000","last_fullsized":"2023-10-09T06:50:44.269747+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":286,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270698+0000","last_change":"2023-10-09T06:49:20.633058+0000","last_active":"2023-10-09T06:50:44.270698+0000","last_peered":"2023-10-09T06:50:44.270698+0000","last_clean":"2023-10-09T06:50:44.270698+0000","last_became_active":"2023-10-09T06:49:20.583223+0000","last_became_peered":"2023-10-09T06:49:20.583223+0000","last_unstale":"2023-10-09T06:50:44.270698+0000","last_undegraded":"2023-10-09T06:50:44.270698+0000","last_fullsized":"2023-10-09T06:50:44.270698+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":357,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280879+0000","last_change":"2023-10-09T06:45:33.535003+0000","last_active":"2023-10-09T06:50:44.280879+0000","last_peered":"2023-10-09T06:50:44.280879+0000","last_clean":"2023-10-09T06:50:44.280879+0000","last_became_active":"2023-10-09T06:45:33.533458+0000","last_became_peered":"2023-10-09T06:45:33.533458+0000","last_unstale":"2023-10-09T06:50:44.280879+0000","last_undegraded":"2023-10-09T06:50:44.280879+0000","last_fullsized":"2023-10-09T06:50:44.280879+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":325,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269929+0000","last_change":"2023-10-09T06:45:33.895855+0000","last_active":"2023-10-09T06:50:44.269929+0000","last_peered":"2023-10-09T06:50:44.269929+0000","last_clean":"2023-10-09T06:50:44.269929+0000","last_became_active":"2023-10-09T06:45:33.538145+0000","last_became_peered":"2023-10-09T06:45:33.538145+0000","last_unstale":"2023-10-09T06:50:44.269929+0000","last_undegraded":"2023-10-09T06:50:44.269929+0000","last_fullsized":"2023-10-09T06:50:44.269929+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":124,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.289945+0000","last_change":"2023-10-09T06:48:57.570316+0000","last_active":"2023-10-09T06:50:44.289945+0000","last_peered":"2023-10-09T06:50:44.289945+0000","last_clean":"2023-10-09T06:50:44.289945+0000","last_became_active":"2023-10-09T06:48:57.568574+0000","last_became_peered":"2023-10-09T06:48:57.568574+0000","last_unstale":"2023-10-09T06:50:44.289945+0000","last_undegraded":"2023-10-09T06:50:44.289945+0000","last_fullsized":"2023-10-09T06:50:44.289945+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280632+0000","last_change":"2023-10-09T06:45:33.529820+0000","last_active":"2023-10-09T06:50:44.280632+0000","last_peered":"2023-10-09T06:50:44.280632+0000","last_clean":"2023-10-09T06:50:44.280632+0000","last_became_active":"2023-10-09T06:45:33.529430+0000","last_became_peered":"2023-10-09T06:45:33.529430+0000","last_unstale":"2023-10-09T06:50:44.280632+0000","last_undegraded":"2023-10-09T06:50:44.280632+0000","last_fullsized":"2023-10-09T06:50:44.280632+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":321,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280508+0000","last_change":"2023-10-09T06:45:33.530387+0000","last_active":"2023-10-09T06:50:44.280508+0000","last_peered":"2023-10-09T06:50:44.280508+0000","last_clean":"2023-10-09T06:50:44.280508+0000","last_became_active":"2023-10-09T06:45:33.530012+0000","last_became_peered":"2023-10-09T06:45:33.530012+0000","last_unstale":"2023-10-09T06:50:44.280508+0000","last_undegraded":"2023-10-09T06:50:44.280508+0000","last_fullsized":"2023-10-09T06:50:44.280508+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":85,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280636+0000","last_change":"2023-10-09T06:45:33.813820+0000","last_active":"2023-10-09T06:50:44.280636+0000","last_peered":"2023-10-09T06:50:44.280636+0000","last_clean":"2023-10-09T06:50:44.280636+0000","last_became_active":"2023-10-09T06:45:33.536995+0000","last_became_peered":"2023-10-09T06:45:33.536995+0000","last_unstale":"2023-10-09T06:50:44.280636+0000","last_undegraded":"2023-10-09T06:50:44.280636+0000","last_fullsized":"2023-10-09T06:50:44.280636+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":360,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280886+0000","last_change":"2023-10-09T06:45:33.534114+0000","last_active":"2023-10-09T06:50:44.280886+0000","last_peered":"2023-10-09T06:50:44.280886+0000","last_clean":"2023-10-09T06:50:44.280886+0000","last_became_active":"2023-10-09T06:45:33.533792+0000","last_became_peered":"2023-10-09T06:45:33.533792+0000","last_unstale":"2023-10-09T06:50:44.280886+0000","last_undegraded":"2023-10-09T06:50:44.280886+0000","last_fullsized":"2023-10-09T06:50:44.280886+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":68,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.268825+0000","last_change":"2023-10-09T06:45:33.544298+0000","last_active":"2023-10-09T06:50:44.268825+0000","last_peered":"2023-10-09T06:50:44.268825+0000","last_clean":"2023-10-09T06:50:44.268825+0000","last_became_active":"2023-10-09T06:45:33.543919+0000","last_became_peered":"2023-10-09T06:45:33.543919+0000","last_unstale":"2023-10-09T06:50:44.268825+0000","last_undegraded":"2023-10-09T06:50:44.268825+0000","last_fullsized":"2023-10-09T06:50:44.268825+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280733+0000","last_change":"2023-10-09T06:49:20.585094+0000","last_active":"2023-10-09T06:50:44.280733+0000","last_peered":"2023-10-09T06:50:44.280733+0000","last_clean":"2023-10-09T06:50:44.280733+0000","last_became_active":"2023-10-09T06:49:20.584741+0000","last_became_peered":"2023-10-09T06:49:20.584741+0000","last_unstale":"2023-10-09T06:50:44.280733+0000","last_undegraded":"2023-10-09T06:50:44.280733+0000","last_fullsized":"2023-10-09T06:50:44.280733+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280338+0000","last_change":"2023-10-09T06:49:20.588455+0000","last_active":"2023-10-09T06:50:44.280338+0000","last_peered":"2023-10-09T06:50:44.280338+0000","last_clean":"2023-10-09T06:50:44.280338+0000","last_became_active":"2023-10-09T06:49:20.587543+0000","last_became_peered":"2023-10-09T06:49:20.587543+0000","last_unstale":"2023-10-09T06:50:44.280338+0000","last_undegraded":"2023-10-09T06:50:44.280338+0000","last_fullsized":"2023-10-09T06:50:44.280338+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":369,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281703+0000","last_change":"2023-10-09T06:45:33.533939+0000","last_active":"2023-10-09T06:50:44.281703+0000","last_peered":"2023-10-09T06:50:44.281703+0000","last_clean":"2023-10-09T06:50:44.281703+0000","last_became_active":"2023-10-09T06:45:33.533600+0000","last_became_peered":"2023-10-09T06:45:33.533600+0000","last_unstale":"2023-10-09T06:50:44.281703+0000","last_undegraded":"2023-10-09T06:50:44.281703+0000","last_fullsized":"2023-10-09T06:50:44.281703+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":327,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269174+0000","last_change":"2023-10-09T06:45:33.534264+0000","last_active":"2023-10-09T06:50:44.269174+0000","last_peered":"2023-10-09T06:50:44.269174+0000","last_clean":"2023-10-09T06:50:44.269174+0000","last_became_active":"2023-10-09T06:45:33.533903+0000","last_became_peered":"2023-10-09T06:45:33.533903+0000","last_unstale":"2023-10-09T06:50:44.269174+0000","last_undegraded":"2023-10-09T06:50:44.269174+0000","last_fullsized":"2023-10-09T06:50:44.269174+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":179,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281592+0000","last_change":"2023-10-09T06:45:33.529126+0000","last_active":"2023-10-09T06:50:44.281592+0000","last_peered":"2023-10-09T06:50:44.281592+0000","last_clean":"2023-10-09T06:50:44.281592+0000","last_became_active":"2023-10-09T06:45:33.528749+0000","last_became_peered":"2023-10-09T06:45:33.528749+0000","last_unstale":"2023-10-09T06:50:44.281592+0000","last_undegraded":"2023-10-09T06:50:44.281592+0000","last_fullsized":"2023-10-09T06:50:44.281592+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"350'344","reported_seq":1845,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.502428+0000","last_change":"2023-10-09T06:50:42.203403+0000","last_active":"2023-10-09T06:50:49.502428+0000","last_peered":"2023-10-09T06:50:49.502428+0000","last_clean":"2023-10-09T06:50:49.502428+0000","last_became_active":"2023-10-09T06:45:37.930357+0000","last_became_peered":"2023-10-09T06:45:37.930357+0000","last_unstale":"2023-10-09T06:50:49.502428+0000","last_undegraded":"2023-10-09T06:50:49.502428+0000","last_fullsized":"2023-10-09T06:50:49.502428+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":344,"log_dups_size":0,"ondisk_log_size":344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2584,"num_read_kb":4613,"num_write":301,"num_write_kb":2356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":231,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269609+0000","last_change":"2023-10-09T06:45:33.535684+0000","last_active":"2023-10-09T06:50:44.269609+0000","last_peered":"2023-10-09T06:50:44.269609+0000","last_clean":"2023-10-09T06:50:44.269609+0000","last_became_active":"2023-10-09T06:45:33.535320+0000","last_became_peered":"2023-10-09T06:45:33.535320+0000","last_unstale":"2023-10-09T06:50:44.269609+0000","last_undegraded":"2023-10-09T06:50:44.269609+0000","last_fullsized":"2023-10-09T06:50:44.269609+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280333+0000","last_change":"2023-10-09T06:49:20.584923+0000","last_active":"2023-10-09T06:50:44.280333+0000","last_peered":"2023-10-09T06:50:44.280333+0000","last_clean":"2023-10-09T06:50:44.280333+0000","last_became_active":"2023-10-09T06:49:20.584322+0000","last_became_peered":"2023-10-09T06:49:20.584322+0000","last_unstale":"2023-10-09T06:50:44.280333+0000","last_undegraded":"2023-10-09T06:50:44.280333+0000","last_fullsized":"2023-10-09T06:50:44.280333+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"350'187","reported_seq":595,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.501846+0000","last_change":"2023-10-09T06:50:42.203305+0000","last_active":"2023-10-09T06:50:49.501846+0000","last_peered":"2023-10-09T06:50:49.501846+0000","last_clean":"2023-10-09T06:50:49.501846+0000","last_became_active":"2023-10-09T06:45:33.543602+0000","last_became_peered":"2023-10-09T06:45:33.543602+0000","last_unstale":"2023-10-09T06:50:49.501846+0000","last_undegraded":"2023-10-09T06:50:49.501846+0000","last_fullsized":"2023-10-09T06:50:49.501846+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":187,"log_dups_size":0,"ondisk_log_size":187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1645,"num_read_kb":3152,"num_write":176,"num_write_kb":2470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":361,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281049+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:50:44.281049+0000","last_peered":"2023-10-09T06:50:44.281049+0000","last_clean":"2023-10-09T06:50:44.281049+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:50:44.281049+0000","last_undegraded":"2023-10-09T06:50:44.281049+0000","last_fullsized":"2023-10-09T06:50:44.281049+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.290315+0000","last_change":"2023-10-09T06:48:57.565279+0000","last_active":"2023-10-09T06:50:44.290315+0000","last_peered":"2023-10-09T06:50:44.290315+0000","last_clean":"2023-10-09T06:50:44.290315+0000","last_became_active":"2023-10-09T06:48:57.564796+0000","last_became_peered":"2023-10-09T06:48:57.564796+0000","last_unstale":"2023-10-09T06:50:44.290315+0000","last_undegraded":"2023-10-09T06:50:44.290315+0000","last_fullsized":"2023-10-09T06:50:44.290315+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"350'186","reported_seq":734,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:50.054454+0000","last_change":"2023-10-09T06:50:42.210920+0000","last_active":"2023-10-09T06:50:50.054454+0000","last_peered":"2023-10-09T06:50:50.054454+0000","last_clean":"2023-10-09T06:50:50.054454+0000","last_became_active":"2023-10-09T06:45:36.692765+0000","last_became_peered":"2023-10-09T06:45:36.692765+0000","last_unstale":"2023-10-09T06:50:50.054454+0000","last_undegraded":"2023-10-09T06:50:50.054454+0000","last_fullsized":"2023-10-09T06:50:50.054454+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":186,"log_dups_size":0,"ondisk_log_size":186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":933,"num_read_kb":1099,"num_write":169,"num_write_kb":4412,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":310,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.289492+0000","last_change":"2023-10-09T06:45:33.532462+0000","last_active":"2023-10-09T06:50:44.289492+0000","last_peered":"2023-10-09T06:50:44.289492+0000","last_clean":"2023-10-09T06:50:44.289492+0000","last_became_active":"2023-10-09T06:45:33.531726+0000","last_became_peered":"2023-10-09T06:45:33.531726+0000","last_unstale":"2023-10-09T06:50:44.289492+0000","last_undegraded":"2023-10-09T06:50:44.289492+0000","last_fullsized":"2023-10-09T06:50:44.289492+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270908+0000","last_change":"2023-10-09T06:49:20.583160+0000","last_active":"2023-10-09T06:50:44.270908+0000","last_peered":"2023-10-09T06:50:44.270908+0000","last_clean":"2023-10-09T06:50:44.270908+0000","last_became_active":"2023-10-09T06:49:20.582682+0000","last_became_peered":"2023-10-09T06:49:20.582682+0000","last_unstale":"2023-10-09T06:50:44.270908+0000","last_undegraded":"2023-10-09T06:50:44.270908+0000","last_fullsized":"2023-10-09T06:50:44.270908+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"350'307","reported_seq":973,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.501766+0000","last_change":"2023-10-09T06:50:42.202523+0000","last_active":"2023-10-09T06:50:49.501766+0000","last_peered":"2023-10-09T06:50:49.501766+0000","last_clean":"2023-10-09T06:50:49.501766+0000","last_became_active":"2023-10-09T06:45:37.929757+0000","last_became_peered":"2023-10-09T06:45:37.929757+0000","last_unstale":"2023-10-09T06:50:49.501766+0000","last_undegraded":"2023-10-09T06:50:49.501766+0000","last_fullsized":"2023-10-09T06:50:49.501766+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":307,"log_dups_size":0,"ondisk_log_size":307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":752,"num_read_kb":4990,"num_write":302,"num_write_kb":3916,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":646,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.268906+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:50:44.268906+0000","last_peered":"2023-10-09T06:50:44.268906+0000","last_clean":"2023-10-09T06:50:44.268906+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:50:44.268906+0000","last_undegraded":"2023-10-09T06:50:44.268906+0000","last_fullsized":"2023-10-09T06:50:44.268906+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":342,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.290170+0000","last_change":"2023-10-09T06:45:33.532290+0000","last_active":"2023-10-09T06:50:44.290170+0000","last_peered":"2023-10-09T06:50:44.290170+0000","last_clean":"2023-10-09T06:50:44.290170+0000","last_became_active":"2023-10-09T06:45:33.531814+0000","last_became_peered":"2023-10-09T06:45:33.531814+0000","last_unstale":"2023-10-09T06:50:44.290170+0000","last_undegraded":"2023-10-09T06:50:44.290170+0000","last_fullsized":"2023-10-09T06:50:44.290170+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281366+0000","last_change":"2023-10-09T06:42:32.498269+0000","last_active":"2023-10-09T06:50:44.281366+0000","last_peered":"2023-10-09T06:50:44.281366+0000","last_clean":"2023-10-09T06:50:44.281366+0000","last_became_active":"2023-10-09T06:42:32.497812+0000","last_became_peered":"2023-10-09T06:42:32.497812+0000","last_unstale":"2023-10-09T06:50:44.281366+0000","last_undegraded":"2023-10-09T06:50:44.281366+0000","last_fullsized":"2023-10-09T06:50:44.281366+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"350'198","reported_seq":784,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.501266+0000","last_change":"2023-10-09T06:50:42.205571+0000","last_active":"2023-10-09T06:50:49.501266+0000","last_peered":"2023-10-09T06:50:49.501266+0000","last_clean":"2023-10-09T06:50:49.501266+0000","last_became_active":"2023-10-09T06:45:33.527346+0000","last_became_peered":"2023-10-09T06:45:33.527346+0000","last_unstale":"2023-10-09T06:50:49.501266+0000","last_undegraded":"2023-10-09T06:50:49.501266+0000","last_fullsized":"2023-10-09T06:50:49.501266+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":198,"log_dups_size":0,"ondisk_log_size":198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1395,"num_read_kb":2773,"num_write":167,"num_write_kb":1385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280355+0000","last_change":"2023-10-09T06:49:20.591229+0000","last_active":"2023-10-09T06:50:44.280355+0000","last_peered":"2023-10-09T06:50:44.280355+0000","last_clean":"2023-10-09T06:50:44.280355+0000","last_became_active":"2023-10-09T06:49:20.590718+0000","last_became_peered":"2023-10-09T06:49:20.590718+0000","last_unstale":"2023-10-09T06:50:44.280355+0000","last_undegraded":"2023-10-09T06:50:44.280355+0000","last_fullsized":"2023-10-09T06:50:44.280355+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270470+0000","last_change":"2023-10-09T06:49:20.585713+0000","last_active":"2023-10-09T06:50:44.270470+0000","last_peered":"2023-10-09T06:50:44.270470+0000","last_clean":"2023-10-09T06:50:44.270470+0000","last_became_active":"2023-10-09T06:49:20.585324+0000","last_became_peered":"2023-10-09T06:49:20.585324+0000","last_unstale":"2023-10-09T06:50:44.270470+0000","last_undegraded":"2023-10-09T06:50:44.270470+0000","last_fullsized":"2023-10-09T06:50:44.270470+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"350'194","reported_seq":694,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.502715+0000","last_change":"2023-10-09T06:50:42.202899+0000","last_active":"2023-10-09T06:50:49.502715+0000","last_peered":"2023-10-09T06:50:49.502715+0000","last_clean":"2023-10-09T06:50:49.502715+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:50:49.502715+0000","last_undegraded":"2023-10-09T06:50:49.502715+0000","last_fullsized":"2023-10-09T06:50:49.502715+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1332,"num_read_kb":3158,"num_write":169,"num_write_kb":2120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281196+0000","last_change":"2023-10-09T06:45:05.135956+0000","last_active":"2023-10-09T06:50:44.281196+0000","last_peered":"2023-10-09T06:50:44.281196+0000","last_clean":"2023-10-09T06:50:44.281196+0000","last_became_active":"2023-10-09T06:45:05.135547+0000","last_became_peered":"2023-10-09T06:45:05.135547+0000","last_unstale":"2023-10-09T06:50:44.281196+0000","last_undegraded":"2023-10-09T06:50:44.281196+0000","last_fullsized":"2023-10-09T06:50:44.281196+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269453+0000","last_change":"2023-10-09T06:49:20.583253+0000","last_active":"2023-10-09T06:50:44.269453+0000","last_peered":"2023-10-09T06:50:44.269453+0000","last_clean":"2023-10-09T06:50:44.269453+0000","last_became_active":"2023-10-09T06:49:20.582889+0000","last_became_peered":"2023-10-09T06:49:20.582889+0000","last_unstale":"2023-10-09T06:50:44.269453+0000","last_undegraded":"2023-10-09T06:50:44.269453+0000","last_fullsized":"2023-10-09T06:50:44.269453+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":173,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270914+0000","last_change":"2023-10-09T06:45:33.940589+0000","last_active":"2023-10-09T06:50:44.270914+0000","last_peered":"2023-10-09T06:50:44.270914+0000","last_clean":"2023-10-09T06:50:44.270914+0000","last_became_active":"2023-10-09T06:45:33.541861+0000","last_became_peered":"2023-10-09T06:45:33.541861+0000","last_unstale":"2023-10-09T06:50:44.270914+0000","last_undegraded":"2023-10-09T06:50:44.270914+0000","last_fullsized":"2023-10-09T06:50:44.270914+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"350'241","reported_seq":1185,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:50.056743+0000","last_change":"2023-10-09T06:50:42.212666+0000","last_active":"2023-10-09T06:50:50.056743+0000","last_peered":"2023-10-09T06:50:50.056743+0000","last_clean":"2023-10-09T06:50:50.056743+0000","last_became_active":"2023-10-09T06:45:36.693034+0000","last_became_peered":"2023-10-09T06:45:36.693034+0000","last_unstale":"2023-10-09T06:50:50.056743+0000","last_undegraded":"2023-10-09T06:50:50.056743+0000","last_fullsized":"2023-10-09T06:50:50.056743+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":241,"log_dups_size":0,"ondisk_log_size":241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1357,"num_read_kb":2514,"num_write":167,"num_write_kb":5394,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":190,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.269048+0000","last_change":"2023-10-09T06:48:57.578038+0000","last_active":"2023-10-09T06:50:44.269048+0000","last_peered":"2023-10-09T06:50:44.269048+0000","last_clean":"2023-10-09T06:50:44.269048+0000","last_became_active":"2023-10-09T06:48:57.577546+0000","last_became_peered":"2023-10-09T06:48:57.577546+0000","last_unstale":"2023-10-09T06:50:44.269048+0000","last_undegraded":"2023-10-09T06:50:44.269048+0000","last_fullsized":"2023-10-09T06:50:44.269048+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"350'293","reported_seq":1326,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-10-09T06:50:49.183764+0000","last_change":"2023-10-09T06:50:42.208175+0000","last_active":"2023-10-09T06:50:49.183764+0000","last_peered":"2023-10-09T06:50:49.183764+0000","last_clean":"2023-10-09T06:50:49.183764+0000","last_became_active":"2023-10-09T06:45:36.687770+0000","last_became_peered":"2023-10-09T06:45:36.687770+0000","last_unstale":"2023-10-09T06:50:49.183764+0000","last_undegraded":"2023-10-09T06:50:49.183764+0000","last_fullsized":"2023-10-09T06:50:49.183764+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1929,"num_read_kb":3111,"num_write":239,"num_write_kb":3373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":92,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281808+0000","last_change":"2023-10-09T06:44:32.353758+0000","last_active":"2023-10-09T06:50:44.281808+0000","last_peered":"2023-10-09T06:50:44.281808+0000","last_clean":"2023-10-09T06:50:44.281808+0000","last_became_active":"2023-10-09T06:44:32.353240+0000","last_became_peered":"2023-10-09T06:44:32.353240+0000","last_unstale":"2023-10-09T06:50:44.281808+0000","last_undegraded":"2023-10-09T06:50:44.281808+0000","last_fullsized":"2023-10-09T06:50:44.281808+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":282,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":345,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280695+0000","last_change":"2023-10-09T06:42:32.476668+0000","last_active":"2023-10-09T06:50:44.280695+0000","last_peered":"2023-10-09T06:50:44.280695+0000","last_clean":"2023-10-09T06:50:44.280695+0000","last_became_active":"2023-10-09T06:42:32.476053+0000","last_became_peered":"2023-10-09T06:42:32.476053+0000","last_unstale":"2023-10-09T06:50:44.280695+0000","last_undegraded":"2023-10-09T06:50:44.280695+0000","last_fullsized":"2023-10-09T06:50:44.280695+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":108,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281516+0000","last_change":"2023-10-09T06:48:57.563675+0000","last_active":"2023-10-09T06:50:44.281516+0000","last_peered":"2023-10-09T06:50:44.281516+0000","last_clean":"2023-10-09T06:50:44.281516+0000","last_became_active":"2023-10-09T06:48:57.563237+0000","last_became_peered":"2023-10-09T06:48:57.563237+0000","last_unstale":"2023-10-09T06:50:44.281516+0000","last_undegraded":"2023-10-09T06:50:44.281516+0000","last_fullsized":"2023-10-09T06:50:44.281516+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":332,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281196+0000","last_change":"2023-10-09T06:48:57.566453+0000","last_active":"2023-10-09T06:50:44.281196+0000","last_peered":"2023-10-09T06:50:44.281196+0000","last_clean":"2023-10-09T06:50:44.281196+0000","last_became_active":"2023-10-09T06:48:57.565889+0000","last_became_peered":"2023-10-09T06:48:57.565889+0000","last_unstale":"2023-10-09T06:50:44.281196+0000","last_undegraded":"2023-10-09T06:50:44.281196+0000","last_fullsized":"2023-10-09T06:50:44.281196+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":184,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281323+0000","last_change":"2023-10-09T06:42:32.593843+0000","last_active":"2023-10-09T06:50:44.281323+0000","last_peered":"2023-10-09T06:50:44.281323+0000","last_clean":"2023-10-09T06:50:44.281323+0000","last_became_active":"2023-10-09T06:42:32.483691+0000","last_became_peered":"2023-10-09T06:42:32.483691+0000","last_unstale":"2023-10-09T06:50:44.281323+0000","last_undegraded":"2023-10-09T06:50:44.281323+0000","last_fullsized":"2023-10-09T06:50:44.281323+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280424+0000","last_change":"2023-10-09T06:49:20.584379+0000","last_active":"2023-10-09T06:50:44.280424+0000","last_peered":"2023-10-09T06:50:44.280424+0000","last_clean":"2023-10-09T06:50:44.280424+0000","last_became_active":"2023-10-09T06:49:20.582540+0000","last_became_peered":"2023-10-09T06:49:20.582540+0000","last_unstale":"2023-10-09T06:50:44.280424+0000","last_undegraded":"2023-10-09T06:50:44.280424+0000","last_fullsized":"2023-10-09T06:50:44.280424+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":197,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.281034+0000","last_change":"2023-10-09T06:45:33.762363+0000","last_active":"2023-10-09T06:50:44.281034+0000","last_peered":"2023-10-09T06:50:44.281034+0000","last_clean":"2023-10-09T06:50:44.281034+0000","last_became_active":"2023-10-09T06:45:33.530125+0000","last_became_peered":"2023-10-09T06:45:33.530125+0000","last_unstale":"2023-10-09T06:50:44.281034+0000","last_undegraded":"2023-10-09T06:50:44.281034+0000","last_fullsized":"2023-10-09T06:50:44.281034+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280274+0000","last_change":"2023-10-09T06:49:20.582679+0000","last_active":"2023-10-09T06:50:44.280274+0000","last_peered":"2023-10-09T06:50:44.280274+0000","last_clean":"2023-10-09T06:50:44.280274+0000","last_became_active":"2023-10-09T06:49:20.582285+0000","last_became_peered":"2023-10-09T06:49:20.582285+0000","last_unstale":"2023-10-09T06:50:44.280274+0000","last_undegraded":"2023-10-09T06:50:44.280274+0000","last_fullsized":"2023-10-09T06:50:44.280274+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":381,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.270491+0000","last_change":"2023-10-09T06:45:33.542948+0000","last_active":"2023-10-09T06:50:44.270491+0000","last_peered":"2023-10-09T06:50:44.270491+0000","last_clean":"2023-10-09T06:50:44.270491+0000","last_became_active":"2023-10-09T06:45:33.542464+0000","last_became_peered":"2023-10-09T06:45:33.542464+0000","last_unstale":"2023-10-09T06:50:44.270491+0000","last_undegraded":"2023-10-09T06:50:44.270491+0000","last_fullsized":"2023-10-09T06:50:44.270491+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":37,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280126+0000","last_change":"2023-10-09T06:49:20.586102+0000","last_active":"2023-10-09T06:50:44.280126+0000","last_peered":"2023-10-09T06:50:44.280126+0000","last_clean":"2023-10-09T06:50:44.280126+0000","last_became_active":"2023-10-09T06:49:20.585465+0000","last_became_peered":"2023-10-09T06:49:20.585465+0000","last_unstale":"2023-10-09T06:50:44.280126+0000","last_undegraded":"2023-10-09T06:50:44.280126+0000","last_fullsized":"2023-10-09T06:50:44.280126+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":372,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.291012+0000","last_change":"2023-10-09T06:48:57.573215+0000","last_active":"2023-10-09T06:50:44.291012+0000","last_peered":"2023-10-09T06:50:44.291012+0000","last_clean":"2023-10-09T06:50:44.291012+0000","last_became_active":"2023-10-09T06:48:57.569057+0000","last_became_peered":"2023-10-09T06:48:57.569057+0000","last_unstale":"2023-10-09T06:50:44.291012+0000","last_undegraded":"2023-10-09T06:50:44.291012+0000","last_fullsized":"2023-10-09T06:50:44.291012+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":387,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-10-09T06:50:44.280026+0000","last_change":"2023-10-09T06:48:57.578851+0000","last_active":"2023-10-09T06:50:44.280026+0000","last_peered":"2023-10-09T06:50:44.280026+0000","last_clean":"2023-10-09T06:50:44.280026+0000","last_became_active":"2023-10-09T06:48:57.578423+0000","last_became_peered":"2023-10-09T06:48:57.578423+0000","last_unstale":"2023-10-09T06:50:44.280026+0000","last_undegraded":"2023-10-09T06:50:44.280026+0000","last_fullsized":"2023-10-09T06:50:44.280026+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":31934,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11927,"num_read_kb":25410,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":262,"num_bytes_recovered":4194399,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":33566720,"data_stored":33554489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430834,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909772,"kb_used_data":3648,"kb_used_omap":0,"kb_used_meta":906112,"kb_avail":92835380,"statfs":{"total":95995035648,"available":95063429120,"internally_reserved":0,"allocated":3735552,"data_stored":3091361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927858688},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":2.081,"15min":1.4710000000000001},"min":{"1min":0.435,"5min":0.224,"15min":0.224},"max":{"1min":1.355,"5min":92.066999999999993,"15min":92.066999999999993},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.1259999999999999,"15min":1.4259999999999999},"min":{"1min":0.48099999999999998,"5min":0.27600000000000002,"15min":0.26900000000000002},"max":{"1min":1.2609999999999999,"5min":91.715999999999994,"15min":91.715999999999994},"last":0.94499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.212,"15min":1.466},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":2.0910000000000002,"5min":6.3559999999999999,"15min":13.526999999999999},"last":0.625},{"interface":"front","average":{"1min":0.877,"5min":1.2829999999999999,"15min":1.3720000000000001},"min":{"1min":0.45200000000000001,"5min":0.33000000000000002,"15min":0.27200000000000002},"max":{"1min":2.1110000000000002,"5min":6.4359999999999999,"15min":13.922000000000001},"last":0.78100000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:50:13 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.819,"15min":1.6830000000000001},"min":{"1min":0.48599999999999999,"5min":0.35999999999999999,"15min":0.29099999999999998},"max":{"1min":5.5,"5min":9.0679999999999996,"15min":15.957000000000001},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.27,"15min":1.292},"min":{"1min":0.376,"5min":0.36299999999999999,"15min":0.33100000000000002},"max":{"1min":1.579,"5min":6.3319999999999999,"15min":6.3319999999999999},"last":0.74199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:49:48 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.8979999999999999,"15min":2.278},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.30499999999999999},"max":{"1min":5.8099999999999996,"5min":9.0449999999999999,"15min":32.715000000000003},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.323,"15min":1.8260000000000001},"min":{"1min":0.45400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.0739999999999998,"5min":6.4690000000000003,"15min":32.728999999999999},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.782,"15min":1.2729999999999999},"min":{"1min":0.63200000000000001,"5min":0.27200000000000002,"15min":0.26700000000000002},"max":{"1min":1.319,"5min":12.711,"15min":12.711},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.008,"5min":2.6859999999999999,"15min":1.893},"min":{"1min":0.47099999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.5229999999999999,"5min":11.369999999999999,"15min":16.064},"last":0.97399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.319,"15min":1.2070000000000001},"min":{"1min":0.49299999999999999,"5min":0.35099999999999998,"15min":0.34799999999999998},"max":{"1min":1.4059999999999999,"5min":6.3049999999999997,"15min":10.680999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.2629999999999999,"15min":1.1919999999999999},"min":{"1min":0.63100000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.3169999999999999,"5min":12.675000000000001,"15min":12.675000000000001},"last":1.1060000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.885,"15min":1.419},"min":{"1min":0.70599999999999996,"5min":0.32900000000000001,"15min":0.32200000000000001},"max":{"1min":1.349,"5min":12.640000000000001,"15min":12.640000000000001},"last":1.069},{"interface":"front","average":{"1min":1.0640000000000001,"5min":2.7130000000000001,"15min":1.9279999999999999},"min":{"1min":0.73699999999999999,"5min":0.42999999999999999,"15min":0.378},"max":{"1min":1.373,"5min":14.613,"15min":16.024000000000001},"last":1.4710000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627066,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900164,"kb_used_data":11836,"kb_used_omap":0,"kb_used_meta":888320,"kb_avail":92844988,"statfs":{"total":95995035648,"available":95073267712,"internally_reserved":0,"allocated":12120064,"data_stored":11487196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909639680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":4.4180000000000001,"15min":2.2749999999999999},"min":{"1min":0.38800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.4279999999999999,"5min":120.822,"15min":120.822},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.77900000000000003,"5min":4.7990000000000004,"15min":2.5070000000000001},"min":{"1min":0.49099999999999999,"5min":0.28899999999999998,"15min":0.26800000000000002},"max":{"1min":1.0920000000000001,"5min":120.47199999999999,"15min":120.47199999999999},"last":1}]},{"osd":1,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.26,"15min":1.5009999999999999},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.0869999999999997,"5min":5.6909999999999998,"15min":19.25},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.262,"15min":1.3839999999999999},"min":{"1min":0.497,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":2.8340000000000001,"5min":6.0369999999999999,"15min":7.952},"last":1.258}]},{"osd":2,"last update":"Mon Oct 9 06:50:20 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.26,"15min":2.4449999999999998},"min":{"1min":0.52100000000000002,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":1.5289999999999999,"5min":6.0780000000000003,"15min":271.976},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.7190000000000001,"15min":2.661},"min":{"1min":0.56100000000000005,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":3.7639999999999998,"5min":11.59,"15min":271.863},"last":1.2290000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.2609999999999999,"15min":1.552},"min":{"1min":0.499,"5min":0.36099999999999999,"15min":0.32800000000000001},"max":{"1min":2.0859999999999999,"5min":6.0570000000000004,"15min":22.873999999999999},"last":1.163},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.3300000000000001,"15min":1.702},"min":{"1min":0.65400000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.1230000000000002,"5min":5.7720000000000002,"15min":25.986999999999998},"last":0.96999999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":2.4550000000000001,"15min":1.9850000000000001},"min":{"1min":0.44500000000000001,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":12.212999999999999,"5min":13.053000000000001,"15min":80.231999999999999},"last":1.125},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.085,"15min":1.9159999999999999},"min":{"1min":0.42299999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.0170000000000003,"5min":20.196000000000002,"15min":81.085999999999999},"last":1.089}]},{"osd":5,"last update":"Mon Oct 9 06:49:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.7649999999999999,"15min":1.389},"min":{"1min":0.55000000000000004,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":7.1609999999999996,"5min":11.558,"15min":11.558},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.347,"15min":1.538},"min":{"1min":0.45800000000000002,"5min":0.311,"15min":0.29199999999999998},"max":{"1min":5.0599999999999996,"5min":5.7400000000000002,"15min":11.526},"last":1.298}]},{"osd":7,"last update":"Mon Oct 9 06:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":2.4889999999999999,"15min":1.756},"min":{"1min":0.69899999999999995,"5min":0.36099999999999999,"15min":0.35299999999999998},"max":{"1min":3.9009999999999998,"5min":12.177,"15min":12.177},"last":1.3879999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.734,"15min":1.3660000000000001},"min":{"1min":0.77200000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":1.6850000000000001,"5min":9.5619999999999994,"15min":9.5619999999999994},"last":1.3480000000000001}]}]},{"osd":1,"up_from":0,"seq":399431958976,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":88,"seq":377957122501,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":960380,"kb_used_data":12460,"kb_used_omap":8,"kb_used_meta":947895,"kb_avail":92784772,"statfs":{"total":95995035648,"available":95011606528,"internally_reserved":0,"allocated":12759040,"data_stored":11550194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":970644736},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.71299999999999997,"15min":1.5269999999999999},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.187},"max":{"1min":1.4810000000000001,"5min":9.0670000000000002,"15min":25.084},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.64600000000000002,"15min":1.534},"min":{"1min":0.374,"5min":0.27800000000000002,"15min":0.20899999999999999},"max":{"1min":1.121,"5min":3.8929999999999998,"15min":25.036999999999999},"last":0.66400000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.78800000000000003,"15min":1.3140000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.20599999999999999},"max":{"1min":1.2709999999999999,"5min":9.1539999999999999,"15min":12.154},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.623,"15min":1.3140000000000001},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.215},"max":{"1min":1.1279999999999999,"5min":3.9489999999999998,"15min":15.351000000000001},"last":0.55000000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:49:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":2.2730000000000001},"min":{"1min":0.39100000000000001,"5min":0.27300000000000002,"15min":0.216},"max":{"1min":0.90200000000000002,"5min":2.851,"15min":86.709999999999994},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.69999999999999996,"15min":2.3330000000000002},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.24299999999999999},"max":{"1min":1.274,"5min":3.9940000000000002,"15min":87.109999999999999},"last":0.94699999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.3759999999999999,"15min":1.3839999999999999},"min":{"1min":0.55200000000000005,"5min":0.35899999999999999,"15min":0.309},"max":{"1min":1.5249999999999999,"5min":9.5459999999999994,"15min":9.5459999999999994},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.4219999999999999,"15min":1.3859999999999999},"min":{"1min":0.53500000000000003,"5min":0.377,"15min":0.31},"max":{"1min":1.548,"5min":11.948,"15min":11.948},"last":0.90400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.0669999999999999,"15min":1.3540000000000001},"min":{"1min":0.438,"5min":0.36699999999999999,"15min":0.27800000000000002},"max":{"1min":1.4690000000000001,"5min":11.894,"15min":11.894},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.018,"5min":0.97799999999999998,"15min":1.393},"min":{"1min":0.66000000000000003,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":1.4990000000000001,"5min":4.6520000000000001,"15min":10.419},"last":1.1180000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:50:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.3480000000000001,"15min":1.5529999999999999},"min":{"1min":0.73099999999999998,"5min":0.432,"15min":0.36799999999999999},"max":{"1min":1.3640000000000001,"5min":8.0619999999999994,"15min":10.377000000000001},"last":1.085},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.2709999999999999,"15min":1.409},"min":{"1min":0.64100000000000001,"5min":0.43099999999999999,"15min":0.379},"max":{"1min":1.6000000000000001,"5min":6.1159999999999997,"15min":9.4510000000000005},"last":0.92900000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:50:41 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.5089999999999999,"15min":1.73},"min":{"1min":0.70799999999999996,"5min":0.46200000000000002,"15min":0.40100000000000002},"max":{"1min":1.5800000000000001,"5min":5.9740000000000002,"15min":13.259},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.3939999999999999,"15min":1.599},"min":{"1min":0.72699999999999998,"5min":0.42399999999999999,"15min":0.35299999999999998},"max":{"1min":1.5369999999999999,"5min":5.798,"15min":13.428000000000001},"last":0.96399999999999997}]}]},{"osd":2,"up_from":0,"seq":420906795450,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":442381631924,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019532,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910536,"kb_used_data":11260,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92834616,"statfs":{"total":95995035648,"available":95062646784,"internally_reserved":0,"allocated":11530240,"data_stored":10890136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:50:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":3.871,"15min":2.8940000000000001},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.249},"max":{"1min":18.120999999999999,"5min":91.683999999999997,"15min":91.683999999999997},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":4.1059999999999999,"15min":3},"min":{"1min":0.47299999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":18.158000000000001,"5min":91.653999999999996,"15min":91.653999999999996},"last":0.92700000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:50:45 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.347,"15min":1.23},"min":{"1min":0.58899999999999997,"5min":0.34799999999999998,"15min":0.28399999999999997},"max":{"1min":1.524,"5min":7.0659999999999998,"15min":7.3860000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.0049999999999999,"15min":1.206},"min":{"1min":0.51500000000000001,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":1.111,"5min":4.4779999999999998,"15min":8.2270000000000003},"last":1.0860000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.6830000000000001,"15min":3.2810000000000001},"min":{"1min":0.623,"5min":0.38400000000000001,"15min":0.28599999999999998},"max":{"1min":1.319,"5min":10.739000000000001,"15min":231.393},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.1719999999999999,"15min":3.1059999999999999},"min":{"1min":0.48199999999999998,"5min":0.34100000000000003,"15min":0.29999999999999999},"max":{"1min":2.9169999999999998,"5min":8.7149999999999999,"15min":231.17099999999999},"last":0.99299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:50:45 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.137,"15min":4.2009999999999996},"min":{"1min":0.57899999999999996,"5min":0.376,"15min":0.34100000000000003},"max":{"1min":1.2689999999999999,"5min":8.6509999999999998,"15min":234.62200000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.4790000000000001,"15min":4.3369999999999997},"min":{"1min":0.64000000000000001,"5min":0.40400000000000003,"15min":0.36199999999999999},"max":{"1min":1.6459999999999999,"5min":8.6340000000000003,"15min":234.673},"last":0.96999999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:50:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.4319999999999999,"15min":1.3009999999999999},"min":{"1min":0.47999999999999998,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":4.5410000000000004,"5min":7.0330000000000004,"15min":22.068000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.456,"5min":1.6950000000000001,"15min":1.3999999999999999},"min":{"1min":0.35499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.8609999999999998,"5min":10.707000000000001,"15min":13.095000000000001},"last":1.1779999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:50:36 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":2.6160000000000001,"15min":1.669},"min":{"1min":0.63,"5min":0.437,"15min":0.38100000000000001},"max":{"1min":1.331,"5min":13.163,"15min":13.163},"last":1.256},{"interface":"front","average":{"1min":0.97199999999999998,"5min":2.0190000000000001,"15min":1.413},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.32300000000000001},"max":{"1min":1.571,"5min":10.138999999999999,"15min":10.138999999999999},"last":1.129}]},{"osd":7,"last update":"Mon Oct 9 06:50:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":2.5110000000000001,"15min":1.823},"min":{"1min":0.70199999999999996,"5min":0.312,"15min":0.312},"max":{"1min":1.4099999999999999,"5min":10.688000000000001,"15min":10.688000000000001},"last":1.3180000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.49,"15min":1.3660000000000001},"min":{"1min":0.53200000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.391,"5min":8.9529999999999994,"15min":8.9529999999999994},"last":0.89300000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823299,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:50:55.261 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:50:55.261 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:50:55.261 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T06:50:55.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T06:50:56.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:55 smithi125 bash[77299]: cluster 2023-10-09T06:50:55.505620+0000 mgr.y (mgr.25723) 1398 : cluster 0 pgmap v1548: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:56.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:55 smithi125 bash[80002]: cluster 2023-10-09T06:50:55.505620+0000 mgr.y (mgr.25723) 1398 : cluster 0 pgmap v1548: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:56.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:55 smithi155 bash[59495]: cluster 2023-10-09T06:50:55.505620+0000 mgr.y (mgr.25723) 1398 : cluster 0 pgmap v1548: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:57.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:57 smithi125 bash[80002]: cluster 2023-10-09T06:50:57.506282+0000 mgr.y (mgr.25723) 1399 : cluster 0 pgmap v1549: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:57.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:57 smithi125 bash[77299]: cluster 2023-10-09T06:50:57.506282+0000 mgr.y (mgr.25723) 1399 : cluster 0 pgmap v1549: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:57 smithi155 bash[59495]: cluster 2023-10-09T06:50:57.506282+0000 mgr.y (mgr.25723) 1399 : cluster 0 pgmap v1549: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 526 KiB/s rd, 928 KiB/s wr, 154 op/s 2023-10-09T06:50:58.824 INFO:teuthology.orchestra.run.smithi125.stdout:osd.0: {} 2023-10-09T06:50:58.845 INFO:teuthology.orchestra.run.smithi125.stdout:osd.1: {} 2023-10-09T06:50:58.867 INFO:teuthology.orchestra.run.smithi125.stdout:osd.2: {} 2023-10-09T06:50:58.895 INFO:teuthology.orchestra.run.smithi125.stdout:osd.3: {} 2023-10-09T06:50:58.920 INFO:teuthology.orchestra.run.smithi125.stdout:osd.4: {} 2023-10-09T06:50:58.943 INFO:teuthology.orchestra.run.smithi125.stdout:osd.5: {} 2023-10-09T06:50:58.963 INFO:teuthology.orchestra.run.smithi125.stdout:osd.6: {} 2023-10-09T06:50:58.994 INFO:teuthology.orchestra.run.smithi125.stdout:osd.7: {} 2023-10-09T06:50:58.996 INFO:teuthology.orchestra.run.smithi125.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.996 INFO:teuthology.orchestra.run.smithi125.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.996 INFO:teuthology.orchestra.run.smithi125.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.996 INFO:teuthology.orchestra.run.smithi125.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.997 INFO:teuthology.orchestra.run.smithi125.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.997 INFO:teuthology.orchestra.run.smithi125.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.997 INFO:teuthology.orchestra.run.smithi125.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:50:58.997 INFO:teuthology.orchestra.run.smithi125.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:50:59.117 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:50:58 smithi125 bash[77299]: audit 2023-10-09T06:50:58.797822+0000 mon.a (mon.0) 1733 : audit 0 from='client.? 172.21.15.125:0/2951893974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:59.117 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:50:58 smithi125 bash[80002]: audit 2023-10-09T06:50:58.797822+0000 mon.a (mon.0) 1733 : audit 0 from='client.? 172.21.15.125:0/2951893974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:59.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:50:58 smithi155 bash[59495]: audit 2023-10-09T06:50:58.797822+0000 mon.a (mon.0) 1733 : audit 0 from='client.? 172.21.15.125:0/2951893974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:50:59.268 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:50:59.268 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:51:00.193 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is set 2023-10-09T06:51:00.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[77299]: audit 2023-10-09T06:50:59.449375+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:51:00.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[77299]: cluster 2023-10-09T06:50:59.506776+0000 mgr.y (mgr.25723) 1400 : cluster 0 pgmap v1550: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 509 KiB/s rd, 898 KiB/s wr, 149 op/s 2023-10-09T06:51:00.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[77299]: audit 2023-10-09T06:50:59.994113+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[77299]: audit 2023-10-09T06:51:00.000647+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[80002]: audit 2023-10-09T06:50:59.449375+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:51:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[80002]: cluster 2023-10-09T06:50:59.506776+0000 mgr.y (mgr.25723) 1400 : cluster 0 pgmap v1550: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 509 KiB/s rd, 898 KiB/s wr, 149 op/s 2023-10-09T06:51:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[80002]: audit 2023-10-09T06:50:59.994113+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:00 smithi125 bash[80002]: audit 2023-10-09T06:51:00.000647+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:00 smithi155 bash[59495]: audit 2023-10-09T06:50:59.449375+0000 mon.a (mon.0) 1734 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:51:00.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:00 smithi155 bash[59495]: cluster 2023-10-09T06:50:59.506776+0000 mgr.y (mgr.25723) 1400 : cluster 0 pgmap v1550: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 509 KiB/s rd, 898 KiB/s wr, 149 op/s 2023-10-09T06:51:00.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:00 smithi155 bash[59495]: audit 2023-10-09T06:50:59.994113+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:00 smithi155 bash[59495]: audit 2023-10-09T06:51:00.000647+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:00.736 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:51:01.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[80002]: cluster 2023-10-09T06:51:00.177372+0000 mon.a (mon.0) 1737 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:51:01.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[80002]: cluster 2023-10-09T06:51:00.185382+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:01.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[80002]: audit 2023-10-09T06:51:00.187361+0000 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:51:01.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[80002]: cluster 2023-10-09T06:51:00.187441+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e352: 8 total, 8 up, 4 in 2023-10-09T06:51:01.629 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[80002]: cluster 2023-10-09T06:51:00.480773+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:01.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[77299]: cluster 2023-10-09T06:51:00.177372+0000 mon.a (mon.0) 1737 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:51:01.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[77299]: cluster 2023-10-09T06:51:00.185382+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:01.630 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[77299]: audit 2023-10-09T06:51:00.187361+0000 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:51:01.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[77299]: cluster 2023-10-09T06:51:00.187441+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e352: 8 total, 8 up, 4 in 2023-10-09T06:51:01.631 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[77299]: cluster 2023-10-09T06:51:00.480773+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:01.631 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:01] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:51:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:01 smithi155 bash[59495]: cluster 2023-10-09T06:51:00.177372+0000 mon.a (mon.0) 1737 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:51:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:01 smithi155 bash[59495]: cluster 2023-10-09T06:51:00.185382+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:01 smithi155 bash[59495]: audit 2023-10-09T06:51:00.187361+0000 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.125:0/2971973522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:51:01.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:01 smithi155 bash[59495]: cluster 2023-10-09T06:51:00.187441+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e352: 8 total, 8 up, 4 in 2023-10-09T06:51:01.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:01 smithi155 bash[59495]: cluster 2023-10-09T06:51:00.480773+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:02.055 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:02.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:02 smithi125 bash[77299]: cluster 2023-10-09T06:51:01.507189+0000 mgr.y (mgr.25723) 1401 : cluster 0 pgmap v1552: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:02.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:02 smithi125 bash[80002]: cluster 2023-10-09T06:51:01.507189+0000 mgr.y (mgr.25723) 1401 : cluster 0 pgmap v1552: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:02.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:02 smithi155 bash[59495]: cluster 2023-10-09T06:51:01.507189+0000 mgr.y (mgr.25723) 1401 : cluster 0 pgmap v1552: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:03.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[77299]: cluster 2023-10-09T06:51:01.423726+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[77299]: cluster 2023-10-09T06:51:01.426371+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.470 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[77299]: cluster 2023-10-09T06:51:02.024116+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[80002]: cluster 2023-10-09T06:51:01.423726+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.470 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[80002]: cluster 2023-10-09T06:51:01.426371+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.471 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:03 smithi125 bash[80002]: cluster 2023-10-09T06:51:02.024116+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.633 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:51:03.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:03 smithi155 bash[59495]: cluster 2023-10-09T06:51:01.423726+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:03 smithi155 bash[59495]: cluster 2023-10-09T06:51:01.426371+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:03.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:03 smithi155 bash[59495]: cluster 2023-10-09T06:51:02.024116+0000 osd.4 (osd.4) 297 : cluster 3 failed to encode map e352 with expected crc 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.208454+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.217131+0000 mon.a (mon.0) 1741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: cluster 2023-10-09T06:51:03.507777+0000 mgr.y (mgr.25723) 1402 : cluster 0 pgmap v1553: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.618459+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.623716+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.738729+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.739951+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:51:04.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[80002]: audit 2023-10-09T06:51:03.748117+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.208454+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.217131+0000 mon.a (mon.0) 1741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: cluster 2023-10-09T06:51:03.507777+0000 mgr.y (mgr.25723) 1402 : cluster 0 pgmap v1553: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.618459+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.623716+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.738729+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.739951+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:51:04.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:04 smithi125 bash[77299]: audit 2023-10-09T06:51:03.748117+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.208454+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.217131+0000 mon.a (mon.0) 1741 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: cluster 2023-10-09T06:51:03.507777+0000 mgr.y (mgr.25723) 1402 : cluster 0 pgmap v1553: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.618459+0000 mon.a (mon.0) 1742 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.623716+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.738729+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:51:04.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.739951+0000 mon.a (mon.0) 1745 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:51:04.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:04 smithi155 bash[59495]: audit 2023-10-09T06:51:03.748117+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:05.235 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:05.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:05 smithi125 bash[77299]: cluster 2023-10-09T06:51:05.508453+0000 mgr.y (mgr.25723) 1403 : cluster 0 pgmap v1554: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:05.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:05 smithi125 bash[80002]: cluster 2023-10-09T06:51:05.508453+0000 mgr.y (mgr.25723) 1403 : cluster 0 pgmap v1554: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:05.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:05 smithi155 bash[59495]: cluster 2023-10-09T06:51:05.508453+0000 mgr.y (mgr.25723) 1403 : cluster 0 pgmap v1554: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:07.188 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:07 smithi125 bash[77299]: debug 2023-10-09T06:51:07.125+0000 7ff949fd9700 -1 mon.a@0(leader).osd e352 definitely_dead 0 2023-10-09T06:51:07.190 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.3. 2023-10-09T06:51:07.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:07 smithi125 bash[77299]: audit 2023-10-09T06:51:07.125825+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.125:0/120847349' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:07.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:07 smithi125 bash[77299]: audit 2023-10-09T06:51:07.126485+0000 mon.a (mon.0) 1747 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:07.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:07 smithi125 bash[80002]: audit 2023-10-09T06:51:07.125825+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.125:0/120847349' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:07.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:07 smithi125 bash[80002]: audit 2023-10-09T06:51:07.126485+0000 mon.a (mon.0) 1747 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:07 smithi155 bash[59495]: audit 2023-10-09T06:51:07.125825+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.125:0/120847349' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:07.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:07 smithi155 bash[59495]: audit 2023-10-09T06:51:07.126485+0000 mon.a (mon.0) 1747 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:51:08.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[80002]: cluster 2023-10-09T06:51:07.181703+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[80002]: audit 2023-10-09T06:51:07.183213+0000 mon.a (mon.0) 1748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[80002]: cluster 2023-10-09T06:51:07.183311+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e353: 8 total, 8 up, 5 in 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[80002]: cluster 2023-10-09T06:51:07.509160+0000 mgr.y (mgr.25723) 1404 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[77299]: cluster 2023-10-09T06:51:07.181703+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[77299]: audit 2023-10-09T06:51:07.183213+0000 mon.a (mon.0) 1748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[77299]: cluster 2023-10-09T06:51:07.183311+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e353: 8 total, 8 up, 5 in 2023-10-09T06:51:08.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:08 smithi125 bash[77299]: cluster 2023-10-09T06:51:07.509160+0000 mgr.y (mgr.25723) 1404 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:08 smithi155 bash[59495]: cluster 2023-10-09T06:51:07.181703+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e353 with expected crc 2023-10-09T06:51:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:08 smithi155 bash[59495]: audit 2023-10-09T06:51:07.183213+0000 mon.a (mon.0) 1748 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:51:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:08 smithi155 bash[59495]: cluster 2023-10-09T06:51:07.183311+0000 mon.a (mon.0) 1749 : cluster 0 osdmap e353: 8 total, 8 up, 5 in 2023-10-09T06:51:08.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:08 smithi155 bash[59495]: cluster 2023-10-09T06:51:07.509160+0000 mgr.y (mgr.25723) 1404 : cluster 0 pgmap v1556: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:08.728 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T06:51:09.212 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.194736+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.196332+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e354: 8 total, 8 up, 5 in 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.197187+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.197422+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.197960+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.199449+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.200355+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.201266+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:08.202028+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.192918+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e355: 8 total, 8 up, 5 in 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.194736+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.196332+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e354: 8 total, 8 up, 5 in 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.197187+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.197422+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.197960+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.199449+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.200355+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.201266+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:08.202028+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:09 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.192918+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e355: 8 total, 8 up, 5 in 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.194736+0000 osd.4 (osd.4) 298 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.196332+0000 mon.a (mon.0) 1750 : cluster 0 osdmap e354: 8 total, 8 up, 5 in 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.197187+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.197422+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.197960+0000 osd.4 (osd.4) 299 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.199449+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.200355+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.201266+0000 osd.4 (osd.4) 300 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:08.202028+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e354 with expected crc 2023-10-09T06:51:09.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:09 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.192918+0000 mon.a (mon.0) 1751 : cluster 0 osdmap e355: 8 total, 8 up, 5 in 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.194299+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.194399+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.195363+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.196351+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.198178+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.198626+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.509780+0000 mgr.y (mgr.25723) 1405 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.584377+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[80002]: cluster 2023-10-09T06:51:09.587125+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.194299+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.194399+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.195363+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.196351+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.198178+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.198626+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.509780+0000 mgr.y (mgr.25723) 1405 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:10.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.584377+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:10 smithi125 bash[77299]: cluster 2023-10-09T06:51:09.587125+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.194299+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.194399+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.195363+0000 osd.4 (osd.4) 301 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.196351+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.198178+0000 osd.4 (osd.4) 302 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.198626+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.509780+0000 mgr.y (mgr.25723) 1405 : cluster 0 pgmap v1559: 73 pgs: 73 active+clean; 8.6 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail 2023-10-09T06:51:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.584377+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:10 smithi155 bash[59495]: cluster 2023-10-09T06:51:09.587125+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e355 with expected crc 2023-10-09T06:51:10.993 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:51:10.993 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:51:11.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:11] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.33.4" 2023-10-09T06:51:11.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[77299]: cluster 2023-10-09T06:51:10.211236+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:11.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[77299]: cluster 2023-10-09T06:51:10.214281+0000 mon.a (mon.0) 1752 : cluster 0 osdmap e356: 8 total, 8 up, 5 in 2023-10-09T06:51:11.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[77299]: cluster 2023-10-09T06:51:10.261190+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:11.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[80002]: cluster 2023-10-09T06:51:10.211236+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:11.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[80002]: cluster 2023-10-09T06:51:10.214281+0000 mon.a (mon.0) 1752 : cluster 0 osdmap e356: 8 total, 8 up, 5 in 2023-10-09T06:51:11.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:11 smithi125 bash[80002]: cluster 2023-10-09T06:51:10.261190+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:11.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:11 smithi155 bash[59495]: cluster 2023-10-09T06:51:10.211236+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:11.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:11 smithi155 bash[59495]: cluster 2023-10-09T06:51:10.214281+0000 mon.a (mon.0) 1752 : cluster 0 osdmap e356: 8 total, 8 up, 5 in 2023-10-09T06:51:11.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:11 smithi155 bash[59495]: cluster 2023-10-09T06:51:10.261190+0000 osd.4 (osd.4) 303 : cluster 3 failed to encode map e356 with expected crc 2023-10-09T06:51:12.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.213652+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.216208+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e357: 8 total, 8 up, 5 in 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.217625+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.217695+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.220862+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.222401+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.222556+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.222827+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.224131+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.368732+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[80002]: cluster 2023-10-09T06:51:11.510390+0000 mgr.y (mgr.25723) 1406 : cluster 0 pgmap v1562: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.213652+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.216208+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e357: 8 total, 8 up, 5 in 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.217625+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.217695+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.220862+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.222401+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.222556+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.222827+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.224131+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.368732+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:12 smithi125 bash[77299]: cluster 2023-10-09T06:51:11.510390+0000 mgr.y (mgr.25723) 1406 : cluster 0 pgmap v1562: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.213652+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.216208+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e357: 8 total, 8 up, 5 in 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.217625+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.217695+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.220862+0000 osd.4 (osd.4) 304 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.222401+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.222556+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.222827+0000 osd.4 (osd.4) 305 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.224131+0000 osd.4 (osd.4) 306 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.368732+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e357 with expected crc 2023-10-09T06:51:12.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:12 smithi155 bash[59495]: cluster 2023-10-09T06:51:11.510390+0000 mgr.y (mgr.25723) 1406 : cluster 0 pgmap v1562: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:12.707 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:51:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.223360+0000 mon.a (mon.0) 1754 : cluster 3 Health check failed: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.236607+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.236721+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.238333+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e358: 8 total, 8 up, 5 in 2023-10-09T06:51:13.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.239380+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.239499+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.239819+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.242951+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[77299]: cluster 2023-10-09T06:51:12.242976+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.223360+0000 mon.a (mon.0) 1754 : cluster 3 Health check failed: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.236607+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.236721+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.238333+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e358: 8 total, 8 up, 5 in 2023-10-09T06:51:13.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.239380+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.239499+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.239819+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.242951+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:13 smithi125 bash[80002]: cluster 2023-10-09T06:51:12.242976+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.223360+0000 mon.a (mon.0) 1754 : cluster 3 Health check failed: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.236607+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.236721+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.238333+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e358: 8 total, 8 up, 5 in 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.239380+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.239499+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.239819+0000 osd.4 (osd.4) 307 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.242951+0000 osd.4 (osd.4) 308 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:13 smithi155 bash[59495]: cluster 2023-10-09T06:51:12.242976+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e358 with expected crc 2023-10-09T06:51:13.729 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:51:13.729 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T06:51:13.729 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[77299]: cluster 2023-10-09T06:51:13.240172+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[77299]: cluster 2023-10-09T06:51:13.242220+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e359: 8 total, 8 up, 5 in 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[77299]: cluster 2023-10-09T06:51:13.243503+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[77299]: cluster 2023-10-09T06:51:13.244954+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[77299]: cluster 2023-10-09T06:51:13.511160+0000 mgr.y (mgr.25723) 1407 : cluster 0 pgmap v1565: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[80002]: cluster 2023-10-09T06:51:13.240172+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[80002]: cluster 2023-10-09T06:51:13.242220+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e359: 8 total, 8 up, 5 in 2023-10-09T06:51:14.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[80002]: cluster 2023-10-09T06:51:13.243503+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[80002]: cluster 2023-10-09T06:51:13.244954+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:14 smithi125 bash[80002]: cluster 2023-10-09T06:51:13.511160+0000 mgr.y (mgr.25723) 1407 : cluster 0 pgmap v1565: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:14.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:14 smithi155 bash[59495]: cluster 2023-10-09T06:51:13.240172+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:14 smithi155 bash[59495]: cluster 2023-10-09T06:51:13.242220+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e359: 8 total, 8 up, 5 in 2023-10-09T06:51:14.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:14 smithi155 bash[59495]: cluster 2023-10-09T06:51:13.243503+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:14 smithi155 bash[59495]: cluster 2023-10-09T06:51:13.244954+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e359 with expected crc 2023-10-09T06:51:14.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:14 smithi155 bash[59495]: cluster 2023-10-09T06:51:13.511160+0000 mgr.y (mgr.25723) 1407 : cluster 0 pgmap v1565: 73 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 67 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 86/87 objects degraded (98.851%); 1/87 objects misplaced (1.149%); 44 B/s, 8 objects/s recovering 2023-10-09T06:51:15.265 INFO:teuthology.orchestra.run.smithi125.stderr:noscrub is unset 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: cluster 2023-10-09T06:51:14.253997+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: cluster 2023-10-09T06:51:14.254117+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: cluster 2023-10-09T06:51:14.255904+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e360: 8 total, 8 up, 5 in 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: cluster 2023-10-09T06:51:14.257294+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: audit 2023-10-09T06:51:14.533226+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.125:0/988136210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[77299]: audit 2023-10-09T06:51:14.533839+0000 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:15.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: cluster 2023-10-09T06:51:14.253997+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: cluster 2023-10-09T06:51:14.254117+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: cluster 2023-10-09T06:51:14.255904+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e360: 8 total, 8 up, 5 in 2023-10-09T06:51:15.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: cluster 2023-10-09T06:51:14.257294+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: audit 2023-10-09T06:51:14.533226+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.125:0/988136210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:15.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:15 smithi125 bash[80002]: audit 2023-10-09T06:51:14.533839+0000 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: cluster 2023-10-09T06:51:14.253997+0000 osd.4 (osd.4) 309 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: cluster 2023-10-09T06:51:14.254117+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: cluster 2023-10-09T06:51:14.255904+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e360: 8 total, 8 up, 5 in 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: cluster 2023-10-09T06:51:14.257294+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e360 with expected crc 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: audit 2023-10-09T06:51:14.533226+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.125:0/988136210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:15.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:15 smithi155 bash[59495]: audit 2023-10-09T06:51:14.533839+0000 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:51:16.361 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.247816+0000 mon.a (mon.0) 1759 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.252194+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: audit 2023-10-09T06:51:15.256990+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.257182+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e361: 8 total, 8 up, 5 in 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.456453+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.456864+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.457546+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[77299]: cluster 2023-10-09T06:51:15.511714+0000 mgr.y (mgr.25723) 1408 : cluster 0 pgmap v1568: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 22 objects/s recovering 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.247816+0000 mon.a (mon.0) 1759 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.252194+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: audit 2023-10-09T06:51:15.256990+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.257182+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e361: 8 total, 8 up, 5 in 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.456453+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.456864+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.457546+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:16 smithi125 bash[80002]: cluster 2023-10-09T06:51:15.511714+0000 mgr.y (mgr.25723) 1408 : cluster 0 pgmap v1568: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 22 objects/s recovering 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.247816+0000 mon.a (mon.0) 1759 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.252194+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: audit 2023-10-09T06:51:15.256990+0000 mon.a (mon.0) 1760 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.257182+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e361: 8 total, 8 up, 5 in 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.456453+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.456864+0000 osd.4 (osd.4) 310 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.457546+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e361 with expected crc 2023-10-09T06:51:16.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:16 smithi155 bash[59495]: cluster 2023-10-09T06:51:15.511714+0000 mgr.y (mgr.25723) 1408 : cluster 0 pgmap v1568: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 22 objects/s recovering 2023-10-09T06:51:17.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:17 smithi125 bash[77299]: cluster 2023-10-09T06:51:16.259305+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded) 2023-10-09T06:51:17.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:17 smithi125 bash[77299]: cluster 2023-10-09T06:51:16.259361+0000 mon.a (mon.0) 1763 : cluster 1 Cluster is now healthy 2023-10-09T06:51:17.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:17 smithi125 bash[80002]: cluster 2023-10-09T06:51:16.259305+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded) 2023-10-09T06:51:17.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:17 smithi125 bash[80002]: cluster 2023-10-09T06:51:16.259361+0000 mon.a (mon.0) 1763 : cluster 1 Cluster is now healthy 2023-10-09T06:51:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:17 smithi155 bash[59495]: cluster 2023-10-09T06:51:16.259305+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/87 objects degraded (98.851%), 4 pgs degraded) 2023-10-09T06:51:17.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:17 smithi155 bash[59495]: cluster 2023-10-09T06:51:16.259361+0000 mon.a (mon.0) 1763 : cluster 1 Cluster is now healthy 2023-10-09T06:51:18.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:18 smithi125 bash[77299]: cluster 2023-10-09T06:51:17.512639+0000 mgr.y (mgr.25723) 1409 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.5 MiB/s, 20 objects/s recovering 2023-10-09T06:51:18.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:18 smithi125 bash[80002]: cluster 2023-10-09T06:51:17.512639+0000 mgr.y (mgr.25723) 1409 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.5 MiB/s, 20 objects/s recovering 2023-10-09T06:51:18.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:18 smithi155 bash[59495]: cluster 2023-10-09T06:51:17.512639+0000 mgr.y (mgr.25723) 1409 : cluster 0 pgmap v1569: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.5 MiB/s, 20 objects/s recovering 2023-10-09T06:51:18.816 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:18.817 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[80002]: audit 2023-10-09T06:51:18.624866+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[80002]: audit 2023-10-09T06:51:18.626764+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:19.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[80002]: cluster 2023-10-09T06:51:19.513280+0000 mgr.y (mgr.25723) 1410 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s, 17 objects/s recovering 2023-10-09T06:51:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[77299]: audit 2023-10-09T06:51:18.624866+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[77299]: audit 2023-10-09T06:51:18.626764+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:19.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:19 smithi125 bash[77299]: cluster 2023-10-09T06:51:19.513280+0000 mgr.y (mgr.25723) 1410 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s, 17 objects/s recovering 2023-10-09T06:51:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:19 smithi155 bash[59495]: audit 2023-10-09T06:51:18.624866+0000 mon.a (mon.0) 1764 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:51:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:19 smithi155 bash[59495]: audit 2023-10-09T06:51:18.626764+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:19 smithi155 bash[59495]: cluster 2023-10-09T06:51:19.513280+0000 mgr.y (mgr.25723) 1410 : cluster 0 pgmap v1570: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1.3 MiB/s, 17 objects/s recovering 2023-10-09T06:51:20.844 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:21.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:21] "GET /metrics HTTP/1.1" 200 34193 "" "Prometheus/2.33.4" 2023-10-09T06:51:21.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:21 smithi125 bash[77299]: cluster 2023-10-09T06:51:21.514342+0000 mgr.y (mgr.25723) 1411 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 13 objects/s recovering 2023-10-09T06:51:21.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:21 smithi125 bash[80002]: cluster 2023-10-09T06:51:21.514342+0000 mgr.y (mgr.25723) 1411 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 13 objects/s recovering 2023-10-09T06:51:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:21 smithi155 bash[59495]: cluster 2023-10-09T06:51:21.514342+0000 mgr.y (mgr.25723) 1411 : cluster 0 pgmap v1571: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 1024 KiB/s, 13 objects/s recovering 2023-10-09T06:51:23.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:23 smithi125 bash[77299]: cluster 2023-10-09T06:51:23.514884+0000 mgr.y (mgr.25723) 1412 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 885 KiB/s, 11 objects/s recovering 2023-10-09T06:51:23.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:23 smithi125 bash[80002]: cluster 2023-10-09T06:51:23.514884+0000 mgr.y (mgr.25723) 1412 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 885 KiB/s, 11 objects/s recovering 2023-10-09T06:51:23.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:23 smithi155 bash[59495]: cluster 2023-10-09T06:51:23.514884+0000 mgr.y (mgr.25723) 1412 : cluster 0 pgmap v1572: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 885 KiB/s, 11 objects/s recovering 2023-10-09T06:51:25.200 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:25 smithi125 bash[77299]: debug 2023-10-09T06:51:25.141+0000 7ff949fd9700 -1 mon.a@0(leader).osd e361 definitely_dead 0 2023-10-09T06:51:25.213 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.1. 2023-10-09T06:51:25.387 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:51:25.387 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:51:25.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:25 smithi125 bash[80002]: audit 2023-10-09T06:51:25.143174+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:51:25.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:25 smithi125 bash[77299]: audit 2023-10-09T06:51:25.143174+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:51:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:25 smithi155 bash[59495]: audit 2023-10-09T06:51:25.143174+0000 mon.a (mon.0) 1766 : audit 0 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T06:51:26.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[77299]: cluster 2023-10-09T06:51:25.205978+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[77299]: cluster 2023-10-09T06:51:25.206406+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[77299]: audit 2023-10-09T06:51:25.206535+0000 mon.a (mon.0) 1767 : audit 1 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[77299]: cluster 2023-10-09T06:51:25.206610+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e362: 8 total, 8 up, 6 in 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[77299]: cluster 2023-10-09T06:51:25.515499+0000 mgr.y (mgr.25723) 1413 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[80002]: cluster 2023-10-09T06:51:25.205978+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[80002]: cluster 2023-10-09T06:51:25.206406+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[80002]: audit 2023-10-09T06:51:25.206535+0000 mon.a (mon.0) 1767 : audit 1 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:51:26.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[80002]: cluster 2023-10-09T06:51:25.206610+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e362: 8 total, 8 up, 6 in 2023-10-09T06:51:26.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:26 smithi125 bash[80002]: cluster 2023-10-09T06:51:25.515499+0000 mgr.y (mgr.25723) 1413 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-10-09T06:51:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:26 smithi155 bash[59495]: cluster 2023-10-09T06:51:25.205978+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:26 smithi155 bash[59495]: cluster 2023-10-09T06:51:25.206406+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e362 with expected crc 2023-10-09T06:51:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:26 smithi155 bash[59495]: audit 2023-10-09T06:51:25.206535+0000 mon.a (mon.0) 1767 : audit 1 from='client.? 172.21.15.125:0/14695964' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T06:51:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:26 smithi155 bash[59495]: cluster 2023-10-09T06:51:25.206610+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e362: 8 total, 8 up, 6 in 2023-10-09T06:51:26.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:26 smithi155 bash[59495]: cluster 2023-10-09T06:51:25.515499+0000 mgr.y (mgr.25723) 1413 : cluster 0 pgmap v1574: 73 pgs: 73 active+clean; 8.6 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-10-09T06:51:27.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.213836+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.229 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.216195+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e363: 8 total, 8 up, 6 in 2023-10-09T06:51:27.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.217831+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.218408+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.219900+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.220210+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.230 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.272357+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.213836+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.216195+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e363: 8 total, 8 up, 6 in 2023-10-09T06:51:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.217831+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.218408+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.219900+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.220210+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:27 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.272357+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.213836+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.216195+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e363: 8 total, 8 up, 6 in 2023-10-09T06:51:27.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.217831+0000 osd.4 (osd.4) 311 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.218408+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.219900+0000 osd.4 (osd.4) 312 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.220210+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:27.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:27 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.272357+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:28.080 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:51:28.081 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T06:51:28.081 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:51:28.081 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T06:51:28.081 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:26.220331+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.219187+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.219250+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.221625+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e364: 8 total, 8 up, 6 in 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.453148+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.455442+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[77299]: cluster 2023-10-09T06:51:27.516146+0000 mgr.y (mgr.25723) 1414 : cluster 0 pgmap v1577: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:26.220331+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.219187+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.219250+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.221625+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e364: 8 total, 8 up, 6 in 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.453148+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.455442+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.388 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:28 smithi125 bash[80002]: cluster 2023-10-09T06:51:27.516146+0000 mgr.y (mgr.25723) 1414 : cluster 0 pgmap v1577: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:26.220331+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e363 with expected crc 2023-10-09T06:51:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.219187+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.219250+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.221625+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e364: 8 total, 8 up, 6 in 2023-10-09T06:51:28.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.453148+0000 osd.4 (osd.4) 313 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.455442+0000 osd.4 (osd.4) 314 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:28.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:28 smithi155 bash[59495]: cluster 2023-10-09T06:51:27.516146+0000 mgr.y (mgr.25723) 1414 : cluster 0 pgmap v1577: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:29.255 INFO:teuthology.orchestra.run.smithi125.stderr:nodeep-scrub is unset 2023-10-09T06:51:29.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: cluster 2023-10-09T06:51:28.237199+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e365: 8 total, 8 up, 6 in 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: cluster 2023-10-09T06:51:28.240882+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: cluster 2023-10-09T06:51:28.858914+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: cluster 2023-10-09T06:51:28.859020+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: cluster 2023-10-09T06:51:28.859262+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[77299]: audit 2023-10-09T06:51:28.883066+0000 mon.a (mon.0) 1772 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:51:29.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: cluster 2023-10-09T06:51:28.237199+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e365: 8 total, 8 up, 6 in 2023-10-09T06:51:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: cluster 2023-10-09T06:51:28.240882+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: cluster 2023-10-09T06:51:28.858914+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: cluster 2023-10-09T06:51:28.859020+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: cluster 2023-10-09T06:51:28.859262+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:29.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:29 smithi125 bash[80002]: audit 2023-10-09T06:51:28.883066+0000 mon.a (mon.0) 1772 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: cluster 2023-10-09T06:51:28.237199+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e365: 8 total, 8 up, 6 in 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: cluster 2023-10-09T06:51:28.240882+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: cluster 2023-10-09T06:51:28.858914+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: cluster 2023-10-09T06:51:28.859020+0000 osd.4 (osd.4) 315 : cluster 3 failed to encode map e365 with expected crc 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: cluster 2023-10-09T06:51:28.859262+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e364 with expected crc 2023-10-09T06:51:29.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:29 smithi155 bash[59495]: audit 2023-10-09T06:51:28.883066+0000 mon.a (mon.0) 1772 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:51:30.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[77299]: cluster 2023-10-09T06:51:29.237779+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[77299]: audit 2023-10-09T06:51:29.240032+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[77299]: cluster 2023-10-09T06:51:29.240102+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e366: 8 total, 8 up, 6 in 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[77299]: cluster 2023-10-09T06:51:29.446911+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[77299]: cluster 2023-10-09T06:51:29.516523+0000 mgr.y (mgr.25723) 1415 : cluster 0 pgmap v1580: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[80002]: cluster 2023-10-09T06:51:29.237779+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[80002]: audit 2023-10-09T06:51:29.240032+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:51:30.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[80002]: cluster 2023-10-09T06:51:29.240102+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e366: 8 total, 8 up, 6 in 2023-10-09T06:51:30.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[80002]: cluster 2023-10-09T06:51:29.446911+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:30 smithi125 bash[80002]: cluster 2023-10-09T06:51:29.516523+0000 mgr.y (mgr.25723) 1415 : cluster 0 pgmap v1580: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:30 smithi155 bash[59495]: cluster 2023-10-09T06:51:29.237779+0000 osd.4 (osd.4) 316 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:30 smithi155 bash[59495]: audit 2023-10-09T06:51:29.240032+0000 mon.a (mon.0) 1773 : audit 1 from='client.? 172.21.15.125:0/689303728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:51:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:30 smithi155 bash[59495]: cluster 2023-10-09T06:51:29.240102+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e366: 8 total, 8 up, 6 in 2023-10-09T06:51:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:30 smithi155 bash[59495]: cluster 2023-10-09T06:51:29.446911+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e366 with expected crc 2023-10-09T06:51:30.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:30 smithi155 bash[59495]: cluster 2023-10-09T06:51:29.516523+0000 mgr.y (mgr.25723) 1415 : cluster 0 pgmap v1580: 73 pgs: 5 peering, 68 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:31.617 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:31] "GET /metrics HTTP/1.1" 200 34193 "" "Prometheus/2.33.4" 2023-10-09T06:51:31.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:31 smithi125 bash[77299]: cluster 2023-10-09T06:51:31.517561+0000 mgr.y (mgr.25723) 1416 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 772 KiB/s, 3 objects/s recovering 2023-10-09T06:51:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:31 smithi125 bash[80002]: cluster 2023-10-09T06:51:31.517561+0000 mgr.y (mgr.25723) 1416 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 772 KiB/s, 3 objects/s recovering 2023-10-09T06:51:31.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:31 smithi155 bash[59495]: cluster 2023-10-09T06:51:31.517561+0000 mgr.y (mgr.25723) 1416 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 772 KiB/s, 3 objects/s recovering 2023-10-09T06:51:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:33 smithi125 bash[77299]: cluster 2023-10-09T06:51:33.518157+0000 mgr.y (mgr.25723) 1417 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 650 KiB/s, 3 objects/s recovering 2023-10-09T06:51:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:33 smithi125 bash[80002]: cluster 2023-10-09T06:51:33.518157+0000 mgr.y (mgr.25723) 1417 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 650 KiB/s, 3 objects/s recovering 2023-10-09T06:51:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:33 smithi155 bash[59495]: cluster 2023-10-09T06:51:33.518157+0000 mgr.y (mgr.25723) 1417 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 650 KiB/s, 3 objects/s recovering 2023-10-09T06:51:33.920 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:33.921 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:34.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:34 smithi125 bash[77299]: audit 2023-10-09T06:51:33.622745+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:34.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:34 smithi125 bash[80002]: audit 2023-10-09T06:51:33.622745+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:34 smithi155 bash[59495]: audit 2023-10-09T06:51:33.622745+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:35.759 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:35 smithi125 bash[77299]: cluster 2023-10-09T06:51:35.519252+0000 mgr.y (mgr.25723) 1418 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 512 KiB/s, 2 objects/s recovering 2023-10-09T06:51:35.760 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:35 smithi125 bash[80002]: cluster 2023-10-09T06:51:35.519252+0000 mgr.y (mgr.25723) 1418 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 512 KiB/s, 2 objects/s recovering 2023-10-09T06:51:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:35 smithi155 bash[59495]: cluster 2023-10-09T06:51:35.519252+0000 mgr.y (mgr.25723) 1418 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 512 KiB/s, 2 objects/s recovering 2023-10-09T06:51:37.893 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:37 smithi125 bash[80002]: cluster 2023-10-09T06:51:37.520165+0000 mgr.y (mgr.25723) 1419 : cluster 0 pgmap v1584: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 441 KiB/s, 2 objects/s recovering 2023-10-09T06:51:37.893 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:37 smithi125 bash[77299]: cluster 2023-10-09T06:51:37.520165+0000 mgr.y (mgr.25723) 1419 : cluster 0 pgmap v1584: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 441 KiB/s, 2 objects/s recovering 2023-10-09T06:51:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:37 smithi155 bash[59495]: cluster 2023-10-09T06:51:37.520165+0000 mgr.y (mgr.25723) 1419 : cluster 0 pgmap v1584: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 441 KiB/s, 2 objects/s recovering 2023-10-09T06:51:39.256 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:51:39.256 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:51:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:39 smithi125 bash[77299]: debug 2023-10-09T06:51:39.617+0000 7ff949fd9700 -1 mon.a@0(leader).osd e366 definitely_dead 0 2023-10-09T06:51:40.174 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:51:40.215 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.5. 2023-10-09T06:51:40.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:40 smithi125 bash[77299]: cluster 2023-10-09T06:51:39.520603+0000 mgr.y (mgr.25723) 1420 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 398 KiB/s, 1 objects/s recovering 2023-10-09T06:51:40.446 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:40 smithi125 bash[77299]: audit 2023-10-09T06:51:39.622069+0000 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:51:40.446 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:40 smithi125 bash[80002]: cluster 2023-10-09T06:51:39.520603+0000 mgr.y (mgr.25723) 1420 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 398 KiB/s, 1 objects/s recovering 2023-10-09T06:51:40.446 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:40 smithi125 bash[80002]: audit 2023-10-09T06:51:39.622069+0000 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:51:40.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:40 smithi155 bash[59495]: cluster 2023-10-09T06:51:39.520603+0000 mgr.y (mgr.25723) 1420 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 398 KiB/s, 1 objects/s recovering 2023-10-09T06:51:40.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:40 smithi155 bash[59495]: audit 2023-10-09T06:51:39.622069+0000 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:51:41.458 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T06:51:41.458 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:51:41.458 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T06:51:41.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T06:51:41.510 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:41] "GET /metrics HTTP/1.1" 200 34204 "" "Prometheus/2.33.4" 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[77299]: cluster 2023-10-09T06:51:40.204361+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[77299]: cluster 2023-10-09T06:51:40.207453+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[77299]: cluster 2023-10-09T06:51:40.207830+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[77299]: audit 2023-10-09T06:51:40.207847+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[77299]: cluster 2023-10-09T06:51:40.207944+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T06:51:41.511 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[80002]: cluster 2023-10-09T06:51:40.204361+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[80002]: cluster 2023-10-09T06:51:40.207453+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[80002]: cluster 2023-10-09T06:51:40.207830+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[80002]: audit 2023-10-09T06:51:40.207847+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:51:41.512 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:41 smithi125 bash[80002]: cluster 2023-10-09T06:51:40.207944+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T06:51:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:41 smithi155 bash[59495]: cluster 2023-10-09T06:51:40.204361+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:41 smithi155 bash[59495]: cluster 2023-10-09T06:51:40.207453+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:41 smithi155 bash[59495]: cluster 2023-10-09T06:51:40.207830+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e367 with expected crc 2023-10-09T06:51:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:41 smithi155 bash[59495]: audit 2023-10-09T06:51:40.207847+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.125:0/3594249901' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:51:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:41 smithi155 bash[59495]: cluster 2023-10-09T06:51:40.207944+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e367: 8 total, 8 up, 7 in 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.206554+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.208128+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.209433+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.209847+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.211949+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.213021+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.213385+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.217928+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.219608+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.221451+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[80002]: cluster 2023-10-09T06:51:41.521105+0000 mgr.y (mgr.25723) 1421 : cluster 0 pgmap v1588: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.206554+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.208128+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T06:51:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.209433+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.209847+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.211949+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.213021+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.213385+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.217928+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.219608+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.221451+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:42 smithi125 bash[77299]: cluster 2023-10-09T06:51:41.521105+0000 mgr.y (mgr.25723) 1421 : cluster 0 pgmap v1588: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.206554+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.208128+0000 mon.a (mon.0) 1779 : cluster 0 osdmap e368: 8 total, 8 up, 7 in 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.209433+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.209847+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.211949+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.213021+0000 osd.4 (osd.4) 317 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.213385+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.217928+0000 osd.4 (osd.4) 318 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.219608+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.221451+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e368 with expected crc 2023-10-09T06:51:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:42 smithi155 bash[59495]: cluster 2023-10-09T06:51:41.521105+0000 mgr.y (mgr.25723) 1421 : cluster 0 pgmap v1588: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2023-10-09T06:51:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[80002]: cluster 2023-10-09T06:51:42.224884+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[80002]: cluster 2023-10-09T06:51:42.227671+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[80002]: cluster 2023-10-09T06:51:42.230007+0000 mon.a (mon.0) 1780 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[80002]: cluster 2023-10-09T06:51:42.237446+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[80002]: cluster 2023-10-09T06:51:42.261599+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[77299]: cluster 2023-10-09T06:51:42.224884+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[77299]: cluster 2023-10-09T06:51:42.227671+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[77299]: cluster 2023-10-09T06:51:42.230007+0000 mon.a (mon.0) 1780 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[77299]: cluster 2023-10-09T06:51:42.237446+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:43 smithi125 bash[77299]: cluster 2023-10-09T06:51:42.261599+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:43 smithi155 bash[59495]: cluster 2023-10-09T06:51:42.224884+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:43 smithi155 bash[59495]: cluster 2023-10-09T06:51:42.227671+0000 osd.4 (osd.4) 319 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:43 smithi155 bash[59495]: cluster 2023-10-09T06:51:42.230007+0000 mon.a (mon.0) 1780 : cluster 0 osdmap e369: 8 total, 8 up, 7 in 2023-10-09T06:51:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:43 smithi155 bash[59495]: cluster 2023-10-09T06:51:42.237446+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:43 smithi155 bash[59495]: cluster 2023-10-09T06:51:42.261599+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e369 with expected crc 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.234436+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.235877+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.238508+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.244221+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.521736+0000 mgr.y (mgr.25723) 1422 : cluster 0 pgmap v1591: 73 pgs: 1 active+undersized+remapped, 3 peering, 69 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 4 B/s, 3 objects/s recovering 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.657155+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[77299]: cluster 2023-10-09T06:51:43.659876+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.234436+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.235877+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.238508+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.244221+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.521736+0000 mgr.y (mgr.25723) 1422 : cluster 0 pgmap v1591: 73 pgs: 1 active+undersized+remapped, 3 peering, 69 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 4 B/s, 3 objects/s recovering 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.657155+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:44 smithi125 bash[80002]: cluster 2023-10-09T06:51:43.659876+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.234436+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.235877+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e370: 8 total, 8 up, 7 in 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.238508+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.244221+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.521736+0000 mgr.y (mgr.25723) 1422 : cluster 0 pgmap v1591: 73 pgs: 1 active+undersized+remapped, 3 peering, 69 active+clean; 8.6 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 4 B/s, 3 objects/s recovering 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.657155+0000 osd.4 (osd.4) 320 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:44 smithi155 bash[59495]: cluster 2023-10-09T06:51:43.659876+0000 osd.4 (osd.4) 321 : cluster 3 failed to encode map e370 with expected crc 2023-10-09T06:51:44.953 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[77299]: cluster 2023-10-09T06:51:44.244739+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[77299]: cluster 2023-10-09T06:51:44.247208+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[77299]: cluster 2023-10-09T06:51:44.249087+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[77299]: cluster 2023-10-09T06:51:44.870063+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[80002]: cluster 2023-10-09T06:51:44.244739+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[80002]: cluster 2023-10-09T06:51:44.247208+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[80002]: cluster 2023-10-09T06:51:44.249087+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:45 smithi125 bash[80002]: cluster 2023-10-09T06:51:44.870063+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:45 smithi155 bash[59495]: cluster 2023-10-09T06:51:44.244739+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:45 smithi155 bash[59495]: cluster 2023-10-09T06:51:44.247208+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e371: 8 total, 8 up, 7 in 2023-10-09T06:51:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:45 smithi155 bash[59495]: cluster 2023-10-09T06:51:44.249087+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:45 smithi155 bash[59495]: cluster 2023-10-09T06:51:44.870063+0000 osd.4 (osd.4) 322 : cluster 3 failed to encode map e371 with expected crc 2023-10-09T06:51:45.969 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:46.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:46 smithi125 bash[77299]: cluster 2023-10-09T06:51:45.522711+0000 mgr.y (mgr.25723) 1423 : cluster 0 pgmap v1593: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 20 B/s, 4 objects/s recovering 2023-10-09T06:51:46.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:46 smithi125 bash[80002]: cluster 2023-10-09T06:51:45.522711+0000 mgr.y (mgr.25723) 1423 : cluster 0 pgmap v1593: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 20 B/s, 4 objects/s recovering 2023-10-09T06:51:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:46 smithi155 bash[59495]: cluster 2023-10-09T06:51:45.522711+0000 mgr.y (mgr.25723) 1423 : cluster 0 pgmap v1593: 73 pgs: 3 peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 20 B/s, 4 objects/s recovering 2023-10-09T06:51:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:47 smithi125 bash[77299]: cluster 2023-10-09T06:51:47.523470+0000 mgr.y (mgr.25723) 1424 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 14 B/s, 3 objects/s recovering 2023-10-09T06:51:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:47 smithi125 bash[80002]: cluster 2023-10-09T06:51:47.523470+0000 mgr.y (mgr.25723) 1424 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 14 B/s, 3 objects/s recovering 2023-10-09T06:51:47.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:47 smithi155 bash[59495]: cluster 2023-10-09T06:51:47.523470+0000 mgr.y (mgr.25723) 1424 : cluster 0 pgmap v1594: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 14 B/s, 3 objects/s recovering 2023-10-09T06:51:48.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:48 smithi125 bash[80002]: audit 2023-10-09T06:51:48.616978+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:48.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:48 smithi125 bash[77299]: audit 2023-10-09T06:51:48.616978+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:49.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:48 smithi155 bash[59495]: audit 2023-10-09T06:51:48.616978+0000 mon.a (mon.0) 1783 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:51:50.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:50 smithi155 bash[59495]: cluster 2023-10-09T06:51:49.524191+0000 mgr.y (mgr.25723) 1425 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 12 B/s, 2 objects/s recovering 2023-10-09T06:51:50.670 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:50 smithi125 bash[77299]: cluster 2023-10-09T06:51:49.524191+0000 mgr.y (mgr.25723) 1425 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 12 B/s, 2 objects/s recovering 2023-10-09T06:51:50.671 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:50 smithi125 bash[80002]: cluster 2023-10-09T06:51:49.524191+0000 mgr.y (mgr.25723) 1425 : cluster 0 pgmap v1595: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 12 B/s, 2 objects/s recovering 2023-10-09T06:51:51.342 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:51:51.342 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:51:51.377 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:51:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:51:51] "GET /metrics HTTP/1.1" 200 34200 "" "Prometheus/2.33.4" 2023-10-09T06:51:51.627 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:51 smithi125 bash[77299]: cluster 2023-10-09T06:51:51.525131+0000 mgr.y (mgr.25723) 1426 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:51:51.627 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:51 smithi125 bash[80002]: cluster 2023-10-09T06:51:51.525131+0000 mgr.y (mgr.25723) 1426 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:51:51.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:51 smithi125 bash[77299]: debug 2023-10-09T06:51:51.801+0000 7ff949fd9700 -1 mon.a@0(leader).osd e371 definitely_dead 0 2023-10-09T06:51:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:51 smithi155 bash[59495]: cluster 2023-10-09T06:51:51.525131+0000 mgr.y (mgr.25723) 1426 : cluster 0 pgmap v1596: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 8 B/s, 0 objects/s recovering 2023-10-09T06:51:52.458 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:51:52.646 INFO:teuthology.orchestra.run.smithi125.stderr:marked in osd.2. 2023-10-09T06:51:52.860 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:52 smithi125 bash[77299]: audit 2023-10-09T06:51:51.805560+0000 mon.a (mon.0) 1784 : audit 0 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:51:52.860 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:52 smithi125 bash[80002]: audit 2023-10-09T06:51:51.805560+0000 mon.a (mon.0) 1784 : audit 0 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:51:52.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:52 smithi155 bash[59495]: audit 2023-10-09T06:51:51.805560+0000 mon.a (mon.0) 1784 : audit 0 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T06:51:53.617 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T06:51:53.618 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[77299]: cluster 2023-10-09T06:51:52.621417+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[77299]: cluster 2023-10-09T06:51:52.621896+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[77299]: audit 2023-10-09T06:51:52.623760+0000 mon.a (mon.0) 1785 : audit 1 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[77299]: cluster 2023-10-09T06:51:52.623827+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.525781+0000 mgr.y (mgr.25723) 1427 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-10-09T06:51:53.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[80002]: cluster 2023-10-09T06:51:52.621417+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[80002]: cluster 2023-10-09T06:51:52.621896+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[80002]: audit 2023-10-09T06:51:52.623760+0000 mon.a (mon.0) 1785 : audit 1 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:51:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[80002]: cluster 2023-10-09T06:51:52.623827+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:51:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:53 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.525781+0000 mgr.y (mgr.25723) 1427 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-10-09T06:51:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:53 smithi155 bash[59495]: cluster 2023-10-09T06:51:52.621417+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:53 smithi155 bash[59495]: cluster 2023-10-09T06:51:52.621896+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e372 with expected crc 2023-10-09T06:51:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:53 smithi155 bash[59495]: audit 2023-10-09T06:51:52.623760+0000 mon.a (mon.0) 1785 : audit 1 from='client.? 172.21.15.125:0/2115083506' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T06:51:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:53 smithi155 bash[59495]: cluster 2023-10-09T06:51:52.623827+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:51:53.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:53 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.525781+0000 mgr.y (mgr.25723) 1427 : cluster 0 pgmap v1598: 73 pgs: 73 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 7 B/s, 0 objects/s recovering 2023-10-09T06:51:54.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.637068+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:54.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.637136+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:54.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.639784+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:51:54.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.640983+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:54.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.641430+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:54.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.641552+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:54.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:54 smithi155 bash[59495]: cluster 2023-10-09T06:51:53.646521+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.637068+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.637136+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.639784+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.640983+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.641430+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.641552+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[80002]: cluster 2023-10-09T06:51:53.646521+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.637068+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.637136+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.639784+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.640983+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.641430+0000 osd.4 (osd.4) 323 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.641552+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:55.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:54 smithi125 bash[77299]: cluster 2023-10-09T06:51:53.646521+0000 osd.4 (osd.4) 324 : cluster 3 failed to encode map e373 with expected crc 2023-10-09T06:51:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.649244+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.650194+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.650240+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:51:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.651300+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.651514+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.653234+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:54.654577+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.526731+0000 mgr.y (mgr.25723) 1428 : cluster 0 pgmap v1601: 73 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 44/87 objects degraded (50.575%); 97 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.649244+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.650194+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.650240+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.651300+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.651514+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.653234+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:54.654577+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.139 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:55 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.526731+0000 mgr.y (mgr.25723) 1428 : cluster 0 pgmap v1601: 73 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 44/87 objects degraded (50.575%); 97 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.649244+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.650194+0000 osd.4 (osd.4) 325 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.650240+0000 mon.a (mon.0) 1788 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.651300+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.651514+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.653234+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:54.654577+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e374 with expected crc 2023-10-09T06:51:56.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:55 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.526731+0000 mgr.y (mgr.25723) 1428 : cluster 0 pgmap v1601: 73 pgs: 2 activating+undersized+degraded+remapped, 1 remapped+peering, 70 active+clean; 8.6 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 44/87 objects degraded (50.575%); 97 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.647889+0000 mon.a (mon.0) 1789 : cluster 3 Health check failed: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.659408+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.663857+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.665479+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.666506+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.667979+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[77299]: cluster 2023-10-09T06:51:55.992233+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.647889+0000 mon.a (mon.0) 1789 : cluster 3 Health check failed: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.659408+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.663857+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.665479+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.666506+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.667979+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:56 smithi125 bash[80002]: cluster 2023-10-09T06:51:55.992233+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.647889+0000 mon.a (mon.0) 1789 : cluster 3 Health check failed: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.659408+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.663857+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.665479+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.666506+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.667979+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:56 smithi155 bash[59495]: cluster 2023-10-09T06:51:55.992233+0000 osd.4 (osd.4) 326 : cluster 3 failed to encode map e375 with expected crc 2023-10-09T06:51:57.296 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.662163+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.666274+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.667224+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.667555+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.669356+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.670574+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.671530+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.671601+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.673046+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.675533+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:56.864083+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[80002]: cluster 2023-10-09T06:51:57.527321+0000 mgr.y (mgr.25723) 1429 : cluster 0 pgmap v1604: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.662163+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.666274+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.667224+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.667555+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.669356+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.670574+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.671530+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.671601+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.673046+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.675533+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:56.864083+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.139 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:57 smithi125 bash[77299]: cluster 2023-10-09T06:51:57.527321+0000 mgr.y (mgr.25723) 1429 : cluster 0 pgmap v1604: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:51:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.662163+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.666274+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:51:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.667224+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.667555+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.669356+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.670574+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.671530+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.671601+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.673046+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.675533+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:56.864083+0000 osd.4 (osd.4) 327 : cluster 3 failed to encode map e376 with expected crc 2023-10-09T06:51:58.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:57 smithi155 bash[59495]: cluster 2023-10-09T06:51:57.527321+0000 mgr.y (mgr.25723) 1429 : cluster 0 pgmap v1604: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 146 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T06:51:58.311 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:51:59.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[77299]: cluster 2023-10-09T06:51:57.678389+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[77299]: cluster 2023-10-09T06:51:57.681320+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[77299]: cluster 2023-10-09T06:51:57.682522+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[77299]: cluster 2023-10-09T06:51:57.682760+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[80002]: cluster 2023-10-09T06:51:57.678389+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[80002]: cluster 2023-10-09T06:51:57.681320+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[80002]: cluster 2023-10-09T06:51:57.682522+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:58 smithi125 bash[80002]: cluster 2023-10-09T06:51:57.682760+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:58 smithi155 bash[59495]: cluster 2023-10-09T06:51:57.678389+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.146 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:58 smithi155 bash[59495]: cluster 2023-10-09T06:51:57.681320+0000 mon.a (mon.0) 1792 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:51:59.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:58 smithi155 bash[59495]: cluster 2023-10-09T06:51:57.682522+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.147 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:58 smithi155 bash[59495]: cluster 2023-10-09T06:51:57.682760+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e377 with expected crc 2023-10-09T06:51:59.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[77299]: cluster 2023-10-09T06:51:58.685270+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:51:59.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[77299]: cluster 2023-10-09T06:51:58.686796+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:51:59.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[77299]: cluster 2023-10-09T06:51:58.688511+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:51:59.871 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[77299]: cluster 2023-10-09T06:51:59.527950+0000 mgr.y (mgr.25723) 1430 : cluster 0 pgmap v1607: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 70 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 0 B/s, 6 objects/s recovering 2023-10-09T06:51:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[80002]: cluster 2023-10-09T06:51:58.685270+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:51:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[80002]: cluster 2023-10-09T06:51:58.686796+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:51:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[80002]: cluster 2023-10-09T06:51:58.688511+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:51:59.872 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:51:59 smithi125 bash[80002]: cluster 2023-10-09T06:51:59.527950+0000 mgr.y (mgr.25723) 1430 : cluster 0 pgmap v1607: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 70 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 0 B/s, 6 objects/s recovering 2023-10-09T06:52:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:59 smithi155 bash[59495]: cluster 2023-10-09T06:51:58.685270+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:52:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:59 smithi155 bash[59495]: cluster 2023-10-09T06:51:58.686796+0000 mon.a (mon.0) 1793 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:52:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:59 smithi155 bash[59495]: cluster 2023-10-09T06:51:58.688511+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e378 with expected crc 2023-10-09T06:52:00.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:51:59 smithi155 bash[59495]: cluster 2023-10-09T06:51:59.527950+0000 mgr.y (mgr.25723) 1430 : cluster 0 pgmap v1607: 73 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 70 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 44/87 objects degraded (50.575%); 0 B/s, 6 objects/s recovering 2023-10-09T06:52:01.622 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:01] "GET /metrics HTTP/1.1" 200 34200 "" "Prometheus/2.33.4" 2023-10-09T06:52:01.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[80002]: cluster 2023-10-09T06:52:01.528977+0000 mgr.y (mgr.25723) 1431 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 11 objects/s recovering 2023-10-09T06:52:01.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[80002]: cluster 2023-10-09T06:52:01.579904+0000 mon.a (mon.0) 1794 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded) 2023-10-09T06:52:01.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[80002]: cluster 2023-10-09T06:52:01.579959+0000 mon.a (mon.0) 1795 : cluster 1 Cluster is now healthy 2023-10-09T06:52:01.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[77299]: cluster 2023-10-09T06:52:01.528977+0000 mgr.y (mgr.25723) 1431 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 11 objects/s recovering 2023-10-09T06:52:01.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[77299]: cluster 2023-10-09T06:52:01.579904+0000 mon.a (mon.0) 1794 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded) 2023-10-09T06:52:01.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:01 smithi125 bash[77299]: cluster 2023-10-09T06:52:01.579959+0000 mon.a (mon.0) 1795 : cluster 1 Cluster is now healthy 2023-10-09T06:52:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:01 smithi155 bash[59495]: cluster 2023-10-09T06:52:01.528977+0000 mgr.y (mgr.25723) 1431 : cluster 0 pgmap v1608: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 11 objects/s recovering 2023-10-09T06:52:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:01 smithi155 bash[59495]: cluster 2023-10-09T06:52:01.579904+0000 mon.a (mon.0) 1794 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/87 objects degraded (50.575%), 2 pgs degraded) 2023-10-09T06:52:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:01 smithi155 bash[59495]: cluster 2023-10-09T06:52:01.579959+0000 mon.a (mon.0) 1795 : cluster 1 Cluster is now healthy 2023-10-09T06:52:03.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:03 smithi125 bash[77299]: cluster 2023-10-09T06:52:03.529620+0000 mgr.y (mgr.25723) 1432 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 10 objects/s recovering 2023-10-09T06:52:03.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:03 smithi125 bash[80002]: cluster 2023-10-09T06:52:03.529620+0000 mgr.y (mgr.25723) 1432 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 10 objects/s recovering 2023-10-09T06:52:03.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:03 smithi155 bash[59495]: cluster 2023-10-09T06:52:03.529620+0000 mgr.y (mgr.25723) 1432 : cluster 0 pgmap v1609: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 10 objects/s recovering 2023-10-09T06:52:04.065 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:52:04.066 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:52:04.280 INFO:teuthology.orchestra.run.smithi125.stdout:mon.a: {} 2023-10-09T06:52:04.297 INFO:teuthology.orchestra.run.smithi125.stdout:mon.b: {} 2023-10-09T06:52:04.314 INFO:teuthology.orchestra.run.smithi125.stdout:mon.c: {} 2023-10-09T06:52:04.316 INFO:teuthology.orchestra.run.smithi125.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:52:04.316 INFO:teuthology.orchestra.run.smithi125.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:52:04.316 INFO:teuthology.orchestra.run.smithi125.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:52:04.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[77299]: audit 2023-10-09T06:52:03.620718+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[77299]: audit 2023-10-09T06:52:03.829615+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:52:04.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[77299]: audit 2023-10-09T06:52:04.260020+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.125:0/244237140' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:52:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[80002]: audit 2023-10-09T06:52:03.620718+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[80002]: audit 2023-10-09T06:52:03.829615+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:52:04.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:04 smithi125 bash[80002]: audit 2023-10-09T06:52:04.260020+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.125:0/244237140' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:52:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:04 smithi155 bash[59495]: audit 2023-10-09T06:52:03.620718+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:04 smithi155 bash[59495]: audit 2023-10-09T06:52:03.829615+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:52:04.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:04 smithi155 bash[59495]: audit 2023-10-09T06:52:04.260020+0000 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.125:0/244237140' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:52:05.690 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:52:05.759 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:52:05.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:05 smithi125 bash[77299]: cluster 2023-10-09T06:52:05.530567+0000 mgr.y (mgr.25723) 1433 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 8 objects/s recovering 2023-10-09T06:52:05.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:05 smithi125 bash[80002]: cluster 2023-10-09T06:52:05.530567+0000 mgr.y (mgr.25723) 1433 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 8 objects/s recovering 2023-10-09T06:52:05.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:05 smithi155 bash[59495]: cluster 2023-10-09T06:52:05.530567+0000 mgr.y (mgr.25723) 1433 : cluster 0 pgmap v1610: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 8 objects/s recovering 2023-10-09T06:52:07.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:07 smithi125 bash[77299]: cluster 2023-10-09T06:52:07.531344+0000 mgr.y (mgr.25723) 1434 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2023-10-09T06:52:07.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:07 smithi125 bash[80002]: cluster 2023-10-09T06:52:07.531344+0000 mgr.y (mgr.25723) 1434 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2023-10-09T06:52:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:07 smithi155 bash[59495]: cluster 2023-10-09T06:52:07.531344+0000 mgr.y (mgr.25723) 1434 : cluster 0 pgmap v1611: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 7 objects/s recovering 2023-10-09T06:52:10.302 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:10.304 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:10.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[77299]: cluster 2023-10-09T06:52:09.532091+0000 mgr.y (mgr.25723) 1435 : cluster 0 pgmap v1612: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-10-09T06:52:10.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[77299]: audit 2023-10-09T06:52:10.022428+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:10.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[77299]: audit 2023-10-09T06:52:10.028314+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[80002]: cluster 2023-10-09T06:52:09.532091+0000 mgr.y (mgr.25723) 1435 : cluster 0 pgmap v1612: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-10-09T06:52:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[80002]: audit 2023-10-09T06:52:10.022428+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:10 smithi125 bash[80002]: audit 2023-10-09T06:52:10.028314+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:10 smithi155 bash[59495]: cluster 2023-10-09T06:52:09.532091+0000 mgr.y (mgr.25723) 1435 : cluster 0 pgmap v1612: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 6 objects/s recovering 2023-10-09T06:52:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:10 smithi155 bash[59495]: audit 2023-10-09T06:52:10.022428+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:10 smithi155 bash[59495]: audit 2023-10-09T06:52:10.028314+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.524 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:11] "GET /metrics HTTP/1.1" 200 34196 "" "Prometheus/2.33.4" 2023-10-09T06:52:11.851 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[80002]: audit 2023-10-09T06:52:10.520780+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.851 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[80002]: audit 2023-10-09T06:52:10.526085+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.851 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[80002]: audit 2023-10-09T06:52:10.948615+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:52:11.851 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[80002]: audit 2023-10-09T06:52:10.950016+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[80002]: audit 2023-10-09T06:52:10.957060+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[77299]: audit 2023-10-09T06:52:10.520780+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[77299]: audit 2023-10-09T06:52:10.526085+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[77299]: audit 2023-10-09T06:52:10.948615+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[77299]: audit 2023-10-09T06:52:10.950016+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:52:11.852 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:11 smithi125 bash[77299]: audit 2023-10-09T06:52:10.957060+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:11 smithi155 bash[59495]: audit 2023-10-09T06:52:10.520780+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:11 smithi155 bash[59495]: audit 2023-10-09T06:52:10.526085+0000 mon.a (mon.0) 1801 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:11 smithi155 bash[59495]: audit 2023-10-09T06:52:10.948615+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:52:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:11 smithi155 bash[59495]: audit 2023-10-09T06:52:10.950016+0000 mon.a (mon.0) 1803 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:52:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:11 smithi155 bash[59495]: audit 2023-10-09T06:52:10.957060+0000 mon.a (mon.0) 1804 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:12.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:12 smithi125 bash[77299]: cluster 2023-10-09T06:52:11.532689+0000 mgr.y (mgr.25723) 1436 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-10-09T06:52:12.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:12 smithi125 bash[80002]: cluster 2023-10-09T06:52:11.532689+0000 mgr.y (mgr.25723) 1436 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-10-09T06:52:12.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:12 smithi155 bash[59495]: cluster 2023-10-09T06:52:11.532689+0000 mgr.y (mgr.25723) 1436 : cluster 0 pgmap v1613: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 3 objects/s recovering 2023-10-09T06:52:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:13 smithi125 bash[77299]: cluster 2023-10-09T06:52:13.533254+0000 mgr.y (mgr.25723) 1437 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:13 smithi125 bash[80002]: cluster 2023-10-09T06:52:13.533254+0000 mgr.y (mgr.25723) 1437 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:13.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:13 smithi155 bash[59495]: cluster 2023-10-09T06:52:13.533254+0000 mgr.y (mgr.25723) 1437 : cluster 0 pgmap v1614: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:15.448 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:52:15.448 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:52:15.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:15 smithi125 bash[80002]: cluster 2023-10-09T06:52:15.533792+0000 mgr.y (mgr.25723) 1438 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:15.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:15 smithi125 bash[77299]: cluster 2023-10-09T06:52:15.533792+0000 mgr.y (mgr.25723) 1438 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:15 smithi155 bash[59495]: cluster 2023-10-09T06:52:15.533792+0000 mgr.y (mgr.25723) 1438 : cluster 0 pgmap v1615: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:16.383 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:52:16.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:16 smithi125 bash[77299]: audit 2023-10-09T06:52:15.745741+0000 mon.a (mon.0) 1805 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:16.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:16 smithi125 bash[80002]: audit 2023-10-09T06:52:15.745741+0000 mon.a (mon.0) 1805 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:16.646 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:52:16.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:16 smithi155 bash[59495]: audit 2023-10-09T06:52:15.745741+0000 mon.a (mon.0) 1805 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:17.639 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[80002]: cluster 2023-10-09T06:52:16.635753+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[80002]: audit 2023-10-09T06:52:16.637376+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[80002]: cluster 2023-10-09T06:52:16.637443+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[80002]: cluster 2023-10-09T06:52:16.639490+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[80002]: cluster 2023-10-09T06:52:17.534389+0000 mgr.y (mgr.25723) 1439 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[77299]: cluster 2023-10-09T06:52:16.635753+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[77299]: audit 2023-10-09T06:52:16.637376+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[77299]: cluster 2023-10-09T06:52:16.637443+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:52:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[77299]: cluster 2023-10-09T06:52:16.639490+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:17 smithi125 bash[77299]: cluster 2023-10-09T06:52:17.534389+0000 mgr.y (mgr.25723) 1439 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:17 smithi155 bash[59495]: cluster 2023-10-09T06:52:16.635753+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:17 smithi155 bash[59495]: audit 2023-10-09T06:52:16.637376+0000 mon.a (mon.0) 1806 : audit 1 from='client.? 172.21.15.125:0/1489933874' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:17 smithi155 bash[59495]: cluster 2023-10-09T06:52:16.637443+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:52:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:17 smithi155 bash[59495]: cluster 2023-10-09T06:52:16.639490+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:17 smithi155 bash[59495]: cluster 2023-10-09T06:52:17.534389+0000 mgr.y (mgr.25723) 1439 : cluster 0 pgmap v1617: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:19.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:18 smithi125 bash[77299]: cluster 2023-10-09T06:52:17.436964+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:19.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:18 smithi125 bash[77299]: audit 2023-10-09T06:52:18.617802+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:19.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:18 smithi125 bash[80002]: cluster 2023-10-09T06:52:17.436964+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:19.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:18 smithi125 bash[80002]: audit 2023-10-09T06:52:18.617802+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:18 smithi155 bash[59495]: cluster 2023-10-09T06:52:17.436964+0000 osd.4 (osd.4) 328 : cluster 3 failed to encode map e379 with expected crc 2023-10-09T06:52:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:18 smithi155 bash[59495]: audit 2023-10-09T06:52:18.617802+0000 mon.a (mon.0) 1808 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[77299]: cluster 2023-10-09T06:52:19.535100+0000 mgr.y (mgr.25723) 1440 : cluster 0 pgmap v1618: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[77299]: audit 2023-10-09T06:52:20.053868+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[77299]: audit 2023-10-09T06:52:20.054442+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[77299]: audit 2023-10-09T06:52:20.054913+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[77299]: audit 2023-10-09T06:52:20.055310+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[80002]: cluster 2023-10-09T06:52:19.535100+0000 mgr.y (mgr.25723) 1440 : cluster 0 pgmap v1618: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[80002]: audit 2023-10-09T06:52:20.053868+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T06:52:20.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[80002]: audit 2023-10-09T06:52:20.054442+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-10-09T06:52:20.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[80002]: audit 2023-10-09T06:52:20.054913+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T06:52:20.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:20 smithi125 bash[80002]: audit 2023-10-09T06:52:20.055310+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-10-09T06:52:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:20 smithi155 bash[59495]: cluster 2023-10-09T06:52:19.535100+0000 mgr.y (mgr.25723) 1440 : cluster 0 pgmap v1618: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:20 smithi155 bash[59495]: audit 2023-10-09T06:52:20.053868+0000 mon.a (mon.0) 1809 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]: dispatch 2023-10-09T06:52:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:20 smithi155 bash[59495]: audit 2023-10-09T06:52:20.054442+0000 mon.a (mon.0) 1810 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-10-09T06:52:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:20 smithi155 bash[59495]: audit 2023-10-09T06:52:20.054913+0000 mon.a (mon.0) 1811 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2023-10-09T06:52:20.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:20 smithi155 bash[59495]: audit 2023-10-09T06:52:20.055310+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-10-09T06:52:21.169 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:21] "GET /metrics HTTP/1.1" 200 34193 "" "Prometheus/2.33.4" 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: cluster 2023-10-09T06:52:20.224574+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: cluster 2023-10-09T06:52:20.227171+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: audit 2023-10-09T06:52:20.229519+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: audit 2023-10-09T06:52:20.229678+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: audit 2023-10-09T06:52:20.229834+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T06:52:21.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: audit 2023-10-09T06:52:20.229961+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[77299]: cluster 2023-10-09T06:52:20.230030+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: cluster 2023-10-09T06:52:20.224574+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: cluster 2023-10-09T06:52:20.227171+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: audit 2023-10-09T06:52:20.229519+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: audit 2023-10-09T06:52:20.229678+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: audit 2023-10-09T06:52:20.229834+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: audit 2023-10-09T06:52:20.229961+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-10-09T06:52:21.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:21 smithi125 bash[80002]: cluster 2023-10-09T06:52:20.230030+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: cluster 2023-10-09T06:52:20.224574+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: cluster 2023-10-09T06:52:20.227171+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e380 with expected crc 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: audit 2023-10-09T06:52:20.229519+0000 mon.a (mon.0) 1813 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: audit 2023-10-09T06:52:20.229678+0000 mon.a (mon.0) 1814 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: audit 2023-10-09T06:52:20.229834+0000 mon.a (mon.0) 1815 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: audit 2023-10-09T06:52:20.229961+0000 mon.a (mon.0) 1816 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-10-09T06:52:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:21 smithi155 bash[59495]: cluster 2023-10-09T06:52:20.230030+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:52:22.185 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:22.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.226594+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.228610+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.229552+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.229731+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.233263+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.237482+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[80002]: cluster 2023-10-09T06:52:21.535832+0000 mgr.y (mgr.25723) 1441 : cluster 0 pgmap v1621: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:22.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.226594+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.228610+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.229552+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.229731+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.233263+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.237482+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:22 smithi125 bash[77299]: cluster 2023-10-09T06:52:21.535832+0000 mgr.y (mgr.25723) 1441 : cluster 0 pgmap v1621: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.226594+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.228610+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:52:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.229552+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.229731+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.233263+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.237482+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e381 with expected crc 2023-10-09T06:52:22.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:22 smithi155 bash[59495]: cluster 2023-10-09T06:52:21.535832+0000 mgr.y (mgr.25723) 1441 : cluster 0 pgmap v1621: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:23.512 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:23 smithi125 bash[77299]: cluster 2023-10-09T06:52:22.234550+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:52:23.513 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:23 smithi125 bash[80002]: cluster 2023-10-09T06:52:22.234550+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:52:23.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:23 smithi155 bash[59495]: cluster 2023-10-09T06:52:22.234550+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:52:24.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:24 smithi125 bash[77299]: cluster 2023-10-09T06:52:23.536389+0000 mgr.y (mgr.25723) 1442 : cluster 0 pgmap v1622: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:24.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:24 smithi125 bash[80002]: cluster 2023-10-09T06:52:23.536389+0000 mgr.y (mgr.25723) 1442 : cluster 0 pgmap v1622: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:24.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:24 smithi155 bash[59495]: cluster 2023-10-09T06:52:23.536389+0000 mgr.y (mgr.25723) 1442 : cluster 0 pgmap v1622: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:25 smithi125 bash[77299]: cluster 2023-10-09T06:52:25.536992+0000 mgr.y (mgr.25723) 1443 : cluster 0 pgmap v1623: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:25 smithi125 bash[80002]: cluster 2023-10-09T06:52:25.536992+0000 mgr.y (mgr.25723) 1443 : cluster 0 pgmap v1623: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:25.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:25 smithi155 bash[59495]: cluster 2023-10-09T06:52:25.536992+0000 mgr.y (mgr.25723) 1443 : cluster 0 pgmap v1623: 73 pgs: 2 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:27.388 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:52:27.389 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:52:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[77299]: cluster 2023-10-09T06:52:27.537674+0000 mgr.y (mgr.25723) 1444 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:27.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[77299]: cluster 2023-10-09T06:52:27.588987+0000 mon.a (mon.0) 1820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:52:27.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[77299]: cluster 2023-10-09T06:52:27.589054+0000 mon.a (mon.0) 1821 : cluster 1 Cluster is now healthy 2023-10-09T06:52:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[80002]: cluster 2023-10-09T06:52:27.537674+0000 mgr.y (mgr.25723) 1444 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[80002]: cluster 2023-10-09T06:52:27.588987+0000 mon.a (mon.0) 1820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:52:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:27 smithi125 bash[80002]: cluster 2023-10-09T06:52:27.589054+0000 mon.a (mon.0) 1821 : cluster 1 Cluster is now healthy 2023-10-09T06:52:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:27 smithi155 bash[59495]: cluster 2023-10-09T06:52:27.537674+0000 mgr.y (mgr.25723) 1444 : cluster 0 pgmap v1624: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:27 smithi155 bash[59495]: cluster 2023-10-09T06:52:27.588987+0000 mon.a (mon.0) 1820 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:52:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:27 smithi155 bash[59495]: cluster 2023-10-09T06:52:27.589054+0000 mon.a (mon.0) 1821 : cluster 1 Cluster is now healthy 2023-10-09T06:52:28.479 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:52:28.653 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T06:52:28.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:28 smithi125 bash[77299]: audit 2023-10-09T06:52:27.753977+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:28.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:28 smithi125 bash[80002]: audit 2023-10-09T06:52:27.753977+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:28.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:28 smithi155 bash[59495]: audit 2023-10-09T06:52:27.753977+0000 mon.a (mon.0) 1822 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:52:29.673 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: audit 2023-10-09T06:52:28.641625+0000 mon.a (mon.0) 1823 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:28.641700+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:28.646714+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:28.647191+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:28.648022+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:28.648523+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.538347+0000 mgr.y (mgr.25723) 1445 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: audit 2023-10-09T06:52:28.641625+0000 mon.a (mon.0) 1823 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:28.641700+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:28.646714+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:28.647191+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:28.648022+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:28.648523+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:29 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.538347+0000 mgr.y (mgr.25723) 1445 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: audit 2023-10-09T06:52:28.641625+0000 mon.a (mon.0) 1823 : audit 1 from='client.? 172.21.15.125:0/1009094571' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:52:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:28.641700+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:52:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:28.646714+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:28.647191+0000 osd.4 (osd.4) 329 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:28.648022+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:28.648523+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e382 with expected crc 2023-10-09T06:52:30.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:29 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.538347+0000 mgr.y (mgr.25723) 1445 : cluster 0 pgmap v1626: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:31.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.637044+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.638826+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:52:31.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.643935+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.644013+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[77299]: cluster 2023-10-09T06:52:29.645190+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.637044+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.638826+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:52:31.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.643935+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.644013+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:30 smithi125 bash[80002]: cluster 2023-10-09T06:52:29.645190+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:30 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.637044+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:30 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.638826+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:52:31.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:30 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.643935+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:30 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.644013+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:30 smithi155 bash[59495]: cluster 2023-10-09T06:52:29.645190+0000 osd.4 (osd.4) 330 : cluster 3 failed to encode map e383 with expected crc 2023-10-09T06:52:31.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:31] "GET /metrics HTTP/1.1" 200 34193 "" "Prometheus/2.33.4" 2023-10-09T06:52:32.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:31 smithi125 bash[77299]: cluster 2023-10-09T06:52:31.539346+0000 mgr.y (mgr.25723) 1446 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:31 smithi125 bash[80002]: cluster 2023-10-09T06:52:31.539346+0000 mgr.y (mgr.25723) 1446 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:32.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:31 smithi155 bash[59495]: cluster 2023-10-09T06:52:31.539346+0000 mgr.y (mgr.25723) 1446 : cluster 0 pgmap v1628: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:33.268 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:33.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:33 smithi125 bash[77299]: cluster 2023-10-09T06:52:33.540159+0000 mgr.y (mgr.25723) 1447 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:33 smithi125 bash[77299]: audit 2023-10-09T06:52:33.617735+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:33 smithi125 bash[80002]: cluster 2023-10-09T06:52:33.540159+0000 mgr.y (mgr.25723) 1447 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:33 smithi125 bash[80002]: audit 2023-10-09T06:52:33.617735+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:33 smithi155 bash[59495]: cluster 2023-10-09T06:52:33.540159+0000 mgr.y (mgr.25723) 1447 : cluster 0 pgmap v1629: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:33 smithi155 bash[59495]: audit 2023-10-09T06:52:33.617735+0000 mon.a (mon.0) 1826 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:34.283 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:35.692 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:35 smithi125 bash[80002]: cluster 2023-10-09T06:52:35.540893+0000 mgr.y (mgr.25723) 1448 : cluster 0 pgmap v1630: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:35.693 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:35 smithi125 bash[77299]: cluster 2023-10-09T06:52:35.540893+0000 mgr.y (mgr.25723) 1448 : cluster 0 pgmap v1630: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:35.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:35 smithi155 bash[59495]: cluster 2023-10-09T06:52:35.540893+0000 mgr.y (mgr.25723) 1448 : cluster 0 pgmap v1630: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:37 smithi125 bash[77299]: cluster 2023-10-09T06:52:37.541675+0000 mgr.y (mgr.25723) 1449 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:37 smithi125 bash[80002]: cluster 2023-10-09T06:52:37.541675+0000 mgr.y (mgr.25723) 1449 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:37.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:37 smithi155 bash[59495]: cluster 2023-10-09T06:52:37.541675+0000 mgr.y (mgr.25723) 1449 : cluster 0 pgmap v1631: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:39.664 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:52:39.664 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:52:40.230 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:52:40.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:40 smithi125 bash[77299]: cluster 2023-10-09T06:52:39.542325+0000 mgr.y (mgr.25723) 1450 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:40.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:40 smithi125 bash[77299]: audit 2023-10-09T06:52:40.119491+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:40.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:40 smithi125 bash[80002]: cluster 2023-10-09T06:52:39.542325+0000 mgr.y (mgr.25723) 1450 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:40.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:40 smithi125 bash[80002]: audit 2023-10-09T06:52:40.119491+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:40.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:40 smithi155 bash[59495]: cluster 2023-10-09T06:52:39.542325+0000 mgr.y (mgr.25723) 1450 : cluster 0 pgmap v1632: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:40.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:40 smithi155 bash[59495]: audit 2023-10-09T06:52:40.119491+0000 mon.a (mon.0) 1827 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:40.987 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:52:41.600 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:41] "GET /metrics HTTP/1.1" 200 34192 "" "Prometheus/2.33.4" 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[77299]: cluster 2023-10-09T06:52:40.217793+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[77299]: audit 2023-10-09T06:52:40.221819+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[77299]: cluster 2023-10-09T06:52:40.221987+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[80002]: cluster 2023-10-09T06:52:40.217793+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[80002]: audit 2023-10-09T06:52:40.221819+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:41.601 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:41 smithi125 bash[80002]: cluster 2023-10-09T06:52:40.221987+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:52:41.602 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:52:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:41 smithi155 bash[59495]: cluster 2023-10-09T06:52:40.217793+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e384 with expected crc 2023-10-09T06:52:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:41 smithi155 bash[59495]: audit 2023-10-09T06:52:40.221819+0000 mon.a (mon.0) 1828 : audit 1 from='client.? 172.21.15.125:0/8391217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:41 smithi155 bash[59495]: cluster 2023-10-09T06:52:40.221987+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:52:42.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.237862+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.239498+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.239753+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.240579+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.241117+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.242728+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.243224+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.244289+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.244584+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.246267+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.246497+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.246980+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.248255+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.249428+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.641 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[77299]: cluster 2023-10-09T06:52:41.542926+0000 mgr.y (mgr.25723) 1451 : cluster 0 pgmap v1635: 73 pgs: 9 peering, 64 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.237862+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.239498+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.239753+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.240579+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.241117+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.242728+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.243224+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.244289+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.244584+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.246267+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.246497+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.246980+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.248255+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.249428+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.643 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:42 smithi125 bash[80002]: cluster 2023-10-09T06:52:41.542926+0000 mgr.y (mgr.25723) 1451 : cluster 0 pgmap v1635: 73 pgs: 9 peering, 64 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.237862+0000 osd.4 (osd.4) 331 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.239498+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:52:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.239753+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.240579+0000 osd.4 (osd.4) 332 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.241117+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.242728+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.243224+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.244289+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.244584+0000 osd.4 (osd.4) 333 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.246267+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.246497+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.246980+0000 osd.4 (osd.4) 334 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.248255+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.249428+0000 osd.4 (osd.4) 335 : cluster 3 failed to encode map e385 with expected crc 2023-10-09T06:52:42.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:42 smithi155 bash[59495]: cluster 2023-10-09T06:52:41.542926+0000 mgr.y (mgr.25723) 1451 : cluster 0 pgmap v1635: 73 pgs: 9 peering, 64 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.249491+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.252764+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.253174+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.254908+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.255474+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.255858+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.258046+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.258215+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[80002]: cluster 2023-10-09T06:52:42.259247+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.249491+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.252764+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.253174+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.254908+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.255474+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.255858+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.258046+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.258215+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:43 smithi125 bash[77299]: cluster 2023-10-09T06:52:42.259247+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.249491+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.252764+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.253174+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.254908+0000 osd.4 (osd.4) 336 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.255474+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.255858+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.258046+0000 osd.4 (osd.4) 337 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.258215+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:43.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:43 smithi155 bash[59495]: cluster 2023-10-09T06:52:42.259247+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e386 with expected crc 2023-10-09T06:52:44.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[77299]: cluster 2023-10-09T06:52:43.543859+0000 mgr.y (mgr.25723) 1452 : cluster 0 pgmap v1637: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 peering, 63 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 0 B/s, 0 objects/s recovering 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.221153+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.222740+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.226841+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[80002]: cluster 2023-10-09T06:52:43.543859+0000 mgr.y (mgr.25723) 1452 : cluster 0 pgmap v1637: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 peering, 63 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 0 B/s, 0 objects/s recovering 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.221153+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.222740+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:44.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:44 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.226841+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:44 smithi155 bash[59495]: cluster 2023-10-09T06:52:43.543859+0000 mgr.y (mgr.25723) 1452 : cluster 0 pgmap v1637: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 9 peering, 63 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 0 B/s, 0 objects/s recovering 2023-10-09T06:52:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:44 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.221153+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:52:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:44 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.222740+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:44.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:44 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.226841+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.222820+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.223456+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.223622+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.224924+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.225814+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.226455+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.228157+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:44.247492+0000 mon.a (mon.0) 1833 : cluster 3 Health check failed: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.233358+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.222820+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.223456+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.223622+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.224924+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.225814+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.226455+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.228157+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:44.247492+0000 mon.a (mon.0) 1833 : cluster 3 Health check failed: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:52:45.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:45 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.233358+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.222820+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.223456+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.223622+0000 osd.4 (osd.4) 338 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.224924+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.225814+0000 osd.4 (osd.4) 339 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.226455+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.228157+0000 osd.4 (osd.4) 340 : cluster 3 failed to encode map e387 with expected crc 2023-10-09T06:52:45.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:44.247492+0000 mon.a (mon.0) 1833 : cluster 3 Health check failed: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:52:45.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:45 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.233358+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:52:45.760 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:45.761 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.230312+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.234742+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.236467+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.237758+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.238064+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.238954+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.239671+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.242039+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:45.544605+0000 mgr.y (mgr.25723) 1453 : cluster 0 pgmap v1640: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 peering, 60 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 951 KiB/s, 12 objects/s recovering 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[77299]: cluster 2023-10-09T06:52:46.235326+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.230312+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.234742+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.236467+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.237758+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.238064+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.238954+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.239671+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.242039+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:45.544605+0000 mgr.y (mgr.25723) 1453 : cluster 0 pgmap v1640: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 peering, 60 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 951 KiB/s, 12 objects/s recovering 2023-10-09T06:52:46.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:46 smithi125 bash[80002]: cluster 2023-10-09T06:52:46.235326+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:52:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.230312+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.234742+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.236467+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.237758+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.238064+0000 osd.4 (osd.4) 341 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.238954+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.239671+0000 osd.4 (osd.4) 342 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.242039+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e388 with expected crc 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:45.544605+0000 mgr.y (mgr.25723) 1453 : cluster 0 pgmap v1640: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 peering, 60 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 951 KiB/s, 12 objects/s recovering 2023-10-09T06:52:46.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:46 smithi155 bash[59495]: cluster 2023-10-09T06:52:46.235326+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[77299]: cluster 2023-10-09T06:52:46.236633+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[77299]: cluster 2023-10-09T06:52:46.239117+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[77299]: cluster 2023-10-09T06:52:46.241341+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[80002]: cluster 2023-10-09T06:52:46.236633+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[80002]: cluster 2023-10-09T06:52:46.239117+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:47 smithi125 bash[80002]: cluster 2023-10-09T06:52:46.241341+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:47 smithi155 bash[59495]: cluster 2023-10-09T06:52:46.236633+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:47 smithi155 bash[59495]: cluster 2023-10-09T06:52:46.239117+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:47.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:47 smithi155 bash[59495]: cluster 2023-10-09T06:52:46.241341+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:48.359 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:48 smithi125 bash[80002]: cluster 2023-10-09T06:52:47.141211+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:48.359 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:48 smithi125 bash[80002]: cluster 2023-10-09T06:52:47.545338+0000 mgr.y (mgr.25723) 1454 : cluster 0 pgmap v1642: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 773 KiB/s, 9 objects/s recovering 2023-10-09T06:52:48.359 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:48 smithi125 bash[77299]: cluster 2023-10-09T06:52:47.141211+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:48.360 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:48 smithi125 bash[77299]: cluster 2023-10-09T06:52:47.545338+0000 mgr.y (mgr.25723) 1454 : cluster 0 pgmap v1642: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 773 KiB/s, 9 objects/s recovering 2023-10-09T06:52:48.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:48 smithi155 bash[59495]: cluster 2023-10-09T06:52:47.141211+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e389 with expected crc 2023-10-09T06:52:48.645 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:48 smithi155 bash[59495]: cluster 2023-10-09T06:52:47.545338+0000 mgr.y (mgr.25723) 1454 : cluster 0 pgmap v1642: 73 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 69 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 23/87 objects degraded (26.437%); 773 KiB/s, 9 objects/s recovering 2023-10-09T06:52:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[77299]: audit 2023-10-09T06:52:48.622383+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[77299]: audit 2023-10-09T06:52:48.623890+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:49.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[77299]: cluster 2023-10-09T06:52:49.546246+0000 mgr.y (mgr.25723) 1455 : cluster 0 pgmap v1643: 73 pgs: 1 active+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 682 KiB/s, 12 objects/s recovering 2023-10-09T06:52:49.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[80002]: audit 2023-10-09T06:52:48.622383+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[80002]: audit 2023-10-09T06:52:48.623890+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:49.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:49 smithi125 bash[80002]: cluster 2023-10-09T06:52:49.546246+0000 mgr.y (mgr.25723) 1455 : cluster 0 pgmap v1643: 73 pgs: 1 active+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 682 KiB/s, 12 objects/s recovering 2023-10-09T06:52:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:49 smithi155 bash[59495]: audit 2023-10-09T06:52:48.622383+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:52:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:49 smithi155 bash[59495]: audit 2023-10-09T06:52:48.623890+0000 mon.a (mon.0) 1837 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:52:49.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:49 smithi155 bash[59495]: cluster 2023-10-09T06:52:49.546246+0000 mgr.y (mgr.25723) 1455 : cluster 0 pgmap v1643: 73 pgs: 1 active+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 682 KiB/s, 12 objects/s recovering 2023-10-09T06:52:50.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:50 smithi125 bash[80002]: cluster 2023-10-09T06:52:50.217100+0000 mon.a (mon.0) 1838 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded) 2023-10-09T06:52:50.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:50 smithi125 bash[80002]: cluster 2023-10-09T06:52:50.217158+0000 mon.a (mon.0) 1839 : cluster 1 Cluster is now healthy 2023-10-09T06:52:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:50 smithi125 bash[77299]: cluster 2023-10-09T06:52:50.217100+0000 mon.a (mon.0) 1838 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded) 2023-10-09T06:52:50.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:50 smithi125 bash[77299]: cluster 2023-10-09T06:52:50.217158+0000 mon.a (mon.0) 1839 : cluster 1 Cluster is now healthy 2023-10-09T06:52:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:50 smithi155 bash[59495]: cluster 2023-10-09T06:52:50.217100+0000 mon.a (mon.0) 1838 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/87 objects degraded (26.437%), 1 pg degraded) 2023-10-09T06:52:50.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:50 smithi155 bash[59495]: cluster 2023-10-09T06:52:50.217158+0000 mon.a (mon.0) 1839 : cluster 1 Cluster is now healthy 2023-10-09T06:52:51.339 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:52:51.340 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:52:51.379 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:52:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:52:51] "GET /metrics HTTP/1.1" 200 34214 "" "Prometheus/2.33.4" 2023-10-09T06:52:51.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:51 smithi125 bash[77299]: cluster 2023-10-09T06:52:51.547368+0000 mgr.y (mgr.25723) 1456 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 559 KiB/s, 10 objects/s recovering 2023-10-09T06:52:51.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:51 smithi125 bash[80002]: cluster 2023-10-09T06:52:51.547368+0000 mgr.y (mgr.25723) 1456 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 559 KiB/s, 10 objects/s recovering 2023-10-09T06:52:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:51 smithi155 bash[59495]: cluster 2023-10-09T06:52:51.547368+0000 mgr.y (mgr.25723) 1456 : cluster 0 pgmap v1644: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 559 KiB/s, 10 objects/s recovering 2023-10-09T06:52:52.354 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:52:52.658 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:52:52.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:52 smithi155 bash[59495]: audit 2023-10-09T06:52:51.689117+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:53.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:52 smithi125 bash[80002]: audit 2023-10-09T06:52:51.689117+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:53.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:52 smithi125 bash[77299]: audit 2023-10-09T06:52:51.689117+0000 mon.a (mon.0) 1840 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:52:53.635 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.644492+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: audit 2023-10-09T06:52:52.647156+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.647258+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.647324+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.647779+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.651512+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:52.652550+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[77299]: cluster 2023-10-09T06:52:53.548161+0000 mgr.y (mgr.25723) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.644492+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: audit 2023-10-09T06:52:52.647156+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.647258+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.647324+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.647779+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.651512+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:52.652550+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:53.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:53 smithi125 bash[80002]: cluster 2023-10-09T06:52:53.548161+0000 mgr.y (mgr.25723) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.644492+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:54.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: audit 2023-10-09T06:52:52.647156+0000 mon.a (mon.0) 1841 : audit 1 from='client.? 172.21.15.125:0/1022540418' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.647258+0000 mon.a (mon.0) 1842 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.647324+0000 osd.4 (osd.4) 343 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.647779+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.651512+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:52.652550+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e390 with expected crc 2023-10-09T06:52:54.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:53 smithi155 bash[59495]: cluster 2023-10-09T06:52:53.548161+0000 mgr.y (mgr.25723) 1457 : cluster 0 pgmap v1646: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:55.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[77299]: cluster 2023-10-09T06:52:53.646999+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[77299]: cluster 2023-10-09T06:52:53.650829+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[77299]: cluster 2023-10-09T06:52:53.656103+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[77299]: cluster 2023-10-09T06:52:53.660723+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[80002]: cluster 2023-10-09T06:52:53.646999+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[80002]: cluster 2023-10-09T06:52:53.650829+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[80002]: cluster 2023-10-09T06:52:53.656103+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:54 smithi125 bash[80002]: cluster 2023-10-09T06:52:53.660723+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:54 smithi155 bash[59495]: cluster 2023-10-09T06:52:53.646999+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:54 smithi155 bash[59495]: cluster 2023-10-09T06:52:53.650829+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:52:55.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:54 smithi155 bash[59495]: cluster 2023-10-09T06:52:53.656103+0000 osd.4 (osd.4) 344 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:55.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:54 smithi155 bash[59495]: cluster 2023-10-09T06:52:53.660723+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:55 smithi125 bash[77299]: cluster 2023-10-09T06:52:54.118301+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:56.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:55 smithi125 bash[77299]: cluster 2023-10-09T06:52:55.549094+0000 mgr.y (mgr.25723) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:56.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:55 smithi125 bash[80002]: cluster 2023-10-09T06:52:54.118301+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:56.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:55 smithi125 bash[80002]: cluster 2023-10-09T06:52:55.549094+0000 mgr.y (mgr.25723) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:56.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:55 smithi155 bash[59495]: cluster 2023-10-09T06:52:54.118301+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e391 with expected crc 2023-10-09T06:52:56.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:55 smithi155 bash[59495]: cluster 2023-10-09T06:52:55.549094+0000 mgr.y (mgr.25723) 1458 : cluster 0 pgmap v1648: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-09T06:52:57.129 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:52:57.633 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:52:57 smithi125 bash[77299]: cluster 2023-10-09T06:52:57.549657+0000 mgr.y (mgr.25723) 1459 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:57.634 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:52:57 smithi125 bash[80002]: cluster 2023-10-09T06:52:57.549657+0000 mgr.y (mgr.25723) 1459 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:57.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:52:57 smithi155 bash[59495]: cluster 2023-10-09T06:52:57.549657+0000 mgr.y (mgr.25723) 1459 : cluster 0 pgmap v1649: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:52:58.145 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:00.498 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:00 smithi125 bash[77299]: cluster 2023-10-09T06:52:59.550526+0000 mgr.y (mgr.25723) 1460 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:00.499 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:00 smithi125 bash[80002]: cluster 2023-10-09T06:52:59.550526+0000 mgr.y (mgr.25723) 1460 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:00.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:00 smithi155 bash[59495]: cluster 2023-10-09T06:52:59.550526+0000 mgr.y (mgr.25723) 1460 : cluster 0 pgmap v1650: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:01.630 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:01] "GET /metrics HTTP/1.1" 200 34214 "" "Prometheus/2.33.4" 2023-10-09T06:53:01.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:01 smithi125 bash[77299]: cluster 2023-10-09T06:53:01.551470+0000 mgr.y (mgr.25723) 1461 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:01.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:01 smithi125 bash[80002]: cluster 2023-10-09T06:53:01.551470+0000 mgr.y (mgr.25723) 1461 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:01.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:01 smithi155 bash[59495]: cluster 2023-10-09T06:53:01.551470+0000 mgr.y (mgr.25723) 1461 : cluster 0 pgmap v1651: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:03.576 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:03.576 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:03.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:03 smithi125 bash[77299]: cluster 2023-10-09T06:53:03.552032+0000 mgr.y (mgr.25723) 1462 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:03.889 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:03 smithi125 bash[77299]: audit 2023-10-09T06:53:03.625212+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:03.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:03 smithi125 bash[80002]: cluster 2023-10-09T06:53:03.552032+0000 mgr.y (mgr.25723) 1462 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:03.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:03 smithi125 bash[80002]: audit 2023-10-09T06:53:03.625212+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:03.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:03 smithi155 bash[59495]: cluster 2023-10-09T06:53:03.552032+0000 mgr.y (mgr.25723) 1462 : cluster 0 pgmap v1652: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:03.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:03 smithi155 bash[59495]: audit 2023-10-09T06:53:03.625212+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:04.648 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:53:04.651 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:53:04.898 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:04 smithi125 bash[77299]: audit 2023-10-09T06:53:03.626849+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:04.898 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:04 smithi125 bash[77299]: audit 2023-10-09T06:53:03.975434+0000 mon.a (mon.0) 1846 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:04.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:04 smithi125 bash[80002]: audit 2023-10-09T06:53:03.626849+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:04.899 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:04 smithi125 bash[80002]: audit 2023-10-09T06:53:03.975434+0000 mon.a (mon.0) 1846 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:04 smithi155 bash[59495]: audit 2023-10-09T06:53:03.626849+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:05.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:04 smithi155 bash[59495]: audit 2023-10-09T06:53:03.975434+0000 mon.a (mon.0) 1846 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:05.727 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[77299]: cluster 2023-10-09T06:53:04.632938+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[77299]: audit 2023-10-09T06:53:04.636427+0000 mon.a (mon.0) 1847 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[77299]: cluster 2023-10-09T06:53:04.636590+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[77299]: cluster 2023-10-09T06:53:05.552860+0000 mgr.y (mgr.25723) 1463 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[80002]: cluster 2023-10-09T06:53:04.632938+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[80002]: audit 2023-10-09T06:53:04.636427+0000 mon.a (mon.0) 1847 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:06.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[80002]: cluster 2023-10-09T06:53:04.636590+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:53:06.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:05 smithi125 bash[80002]: cluster 2023-10-09T06:53:05.552860+0000 mgr.y (mgr.25723) 1463 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:05 smithi155 bash[59495]: cluster 2023-10-09T06:53:04.632938+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:05 smithi155 bash[59495]: audit 2023-10-09T06:53:04.636427+0000 mon.a (mon.0) 1847 : audit 1 from='client.? 172.21.15.125:0/1521618081' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:05 smithi155 bash[59495]: cluster 2023-10-09T06:53:04.636590+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:53:06.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:05 smithi155 bash[59495]: cluster 2023-10-09T06:53:05.552860+0000 mgr.y (mgr.25723) 1463 : cluster 0 pgmap v1654: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:07.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[77299]: cluster 2023-10-09T06:53:05.749692+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[77299]: cluster 2023-10-09T06:53:05.752509+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[77299]: cluster 2023-10-09T06:53:05.910534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[77299]: cluster 2023-10-09T06:53:06.067781+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[80002]: cluster 2023-10-09T06:53:05.749692+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[80002]: cluster 2023-10-09T06:53:05.752509+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[80002]: cluster 2023-10-09T06:53:05.910534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:06 smithi125 bash[80002]: cluster 2023-10-09T06:53:06.067781+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:06 smithi155 bash[59495]: cluster 2023-10-09T06:53:05.749692+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:06 smithi155 bash[59495]: cluster 2023-10-09T06:53:05.752509+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:06 smithi155 bash[59495]: cluster 2023-10-09T06:53:05.910534+0000 osd.4 (osd.4) 345 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:06 smithi155 bash[59495]: cluster 2023-10-09T06:53:06.067781+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e392 with expected crc 2023-10-09T06:53:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:07 smithi125 bash[77299]: cluster 2023-10-09T06:53:07.553712+0000 mgr.y (mgr.25723) 1464 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:08.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:07 smithi125 bash[80002]: cluster 2023-10-09T06:53:07.553712+0000 mgr.y (mgr.25723) 1464 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:08.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:07 smithi155 bash[59495]: cluster 2023-10-09T06:53:07.553712+0000 mgr.y (mgr.25723) 1464 : cluster 0 pgmap v1655: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:09.424 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:10.442 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:10.495 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:10 smithi125 bash[80002]: cluster 2023-10-09T06:53:09.554521+0000 mgr.y (mgr.25723) 1465 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:10.495 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:10 smithi125 bash[77299]: cluster 2023-10-09T06:53:09.554521+0000 mgr.y (mgr.25723) 1465 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:10 smithi155 bash[59495]: cluster 2023-10-09T06:53:09.554521+0000 mgr.y (mgr.25723) 1465 : cluster 0 pgmap v1656: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:11.473 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:11 smithi125 bash[77299]: audit 2023-10-09T06:53:11.032468+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:53:11.473 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:11] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:53:11.474 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:11 smithi125 bash[80002]: audit 2023-10-09T06:53:11.032468+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:53:11.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:11 smithi155 bash[59495]: audit 2023-10-09T06:53:11.032468+0000 mon.a (mon.0) 1849 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:53:12.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:12 smithi125 bash[77299]: cluster 2023-10-09T06:53:11.555488+0000 mgr.y (mgr.25723) 1466 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:12.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:12 smithi125 bash[80002]: cluster 2023-10-09T06:53:11.555488+0000 mgr.y (mgr.25723) 1466 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:12.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:12 smithi155 bash[59495]: cluster 2023-10-09T06:53:11.555488+0000 mgr.y (mgr.25723) 1466 : cluster 0 pgmap v1657: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:13 smithi125 bash[77299]: cluster 2023-10-09T06:53:13.556268+0000 mgr.y (mgr.25723) 1467 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:13 smithi125 bash[80002]: cluster 2023-10-09T06:53:13.556268+0000 mgr.y (mgr.25723) 1467 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:13 smithi155 bash[59495]: cluster 2023-10-09T06:53:13.556268+0000 mgr.y (mgr.25723) 1467 : cluster 0 pgmap v1658: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:15.699 INFO:teuthology.orchestra.run.smithi125.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:15.699 INFO:teuthology.orchestra.run.smithi125.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:15.747 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:15 smithi125 bash[77299]: cluster 2023-10-09T06:53:15.556837+0000 mgr.y (mgr.25723) 1468 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:15.748 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:15 smithi125 bash[80002]: cluster 2023-10-09T06:53:15.556837+0000 mgr.y (mgr.25723) 1468 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:15 smithi155 bash[59495]: cluster 2023-10-09T06:53:15.556837+0000 mgr.y (mgr.25723) 1468 : cluster 0 pgmap v1659: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:16.652 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T06:53:16.720 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:53:16.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:16 smithi125 bash[77299]: audit 2023-10-09T06:53:16.191934+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.125:0/3781032562' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:16.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:16 smithi125 bash[77299]: audit 2023-10-09T06:53:16.192432+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:16.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:16 smithi125 bash[80002]: audit 2023-10-09T06:53:16.191934+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.125:0/3781032562' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:16.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:16 smithi125 bash[80002]: audit 2023-10-09T06:53:16.192432+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:16 smithi155 bash[59495]: audit 2023-10-09T06:53:16.191934+0000 mon.c (mon.2) 143 : audit 1 from='client.? 172.21.15.125:0/3781032562' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:16 smithi155 bash[59495]: audit 2023-10-09T06:53:16.192432+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:53:17.577 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:53:17.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: cluster 2023-10-09T06:53:16.642401+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: audit 2023-10-09T06:53:16.644163+0000 mon.a (mon.0) 1851 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: cluster 2023-10-09T06:53:16.644255+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: audit 2023-10-09T06:53:17.224944+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: audit 2023-10-09T06:53:17.231488+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: cluster 2023-10-09T06:53:17.233969+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[77299]: cluster 2023-10-09T06:53:17.557291+0000 mgr.y (mgr.25723) 1469 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: cluster 2023-10-09T06:53:16.642401+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:17.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: audit 2023-10-09T06:53:16.644163+0000 mon.a (mon.0) 1851 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:17.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: cluster 2023-10-09T06:53:16.644255+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:53:17.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: audit 2023-10-09T06:53:17.224944+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:17.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: audit 2023-10-09T06:53:17.231488+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:17.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: cluster 2023-10-09T06:53:17.233969+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:17.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:17 smithi125 bash[80002]: cluster 2023-10-09T06:53:17.557291+0000 mgr.y (mgr.25723) 1469 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: cluster 2023-10-09T06:53:16.642401+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: audit 2023-10-09T06:53:16.644163+0000 mon.a (mon.0) 1851 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:53:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: cluster 2023-10-09T06:53:16.644255+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:53:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: audit 2023-10-09T06:53:17.224944+0000 mon.a (mon.0) 1853 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:18.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: audit 2023-10-09T06:53:17.231488+0000 mon.a (mon.0) 1854 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:18.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: cluster 2023-10-09T06:53:17.233969+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e393 with expected crc 2023-10-09T06:53:18.145 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:17 smithi155 bash[59495]: cluster 2023-10-09T06:53:17.557291+0000 mgr.y (mgr.25723) 1469 : cluster 0 pgmap v1661: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:19 smithi125 bash[77299]: audit 2023-10-09T06:53:18.619128+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:19 smithi125 bash[80002]: audit 2023-10-09T06:53:18.619128+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:19.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:19 smithi155 bash[59495]: audit 2023-10-09T06:53:18.619128+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:19.440968+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:19.447391+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: cluster 2023-10-09T06:53:19.557919+0000 mgr.y (mgr.25723) 1470 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:19.918718+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:19.920386+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:19.929186+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:20.238074+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[77299]: audit 2023-10-09T06:53:20.238790+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:19.440968+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:19.447391+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: cluster 2023-10-09T06:53:19.557919+0000 mgr.y (mgr.25723) 1470 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:19.918718+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:19.920386+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:53:20.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:19.929186+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:20.238074+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:53:20.889 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:20 smithi125 bash[80002]: audit 2023-10-09T06:53:20.238790+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:19.440968+0000 mon.a (mon.0) 1856 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:19.447391+0000 mon.a (mon.0) 1857 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: cluster 2023-10-09T06:53:19.557919+0000 mgr.y (mgr.25723) 1470 : cluster 0 pgmap v1662: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:19.918718+0000 mon.a (mon.0) 1858 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:19.920386+0000 mon.a (mon.0) 1859 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:19.929186+0000 mon.a (mon.0) 1860 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:20.238074+0000 mon.a (mon.0) 1861 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:53:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:20 smithi155 bash[59495]: audit 2023-10-09T06:53:20.238790+0000 mon.a (mon.0) 1862 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T06:53:21.537 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:53:21.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:21] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: cluster 2023-10-09T06:53:20.934745+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: audit 2023-10-09T06:53:20.934933+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: audit 2023-10-09T06:53:20.935047+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: cluster 2023-10-09T06:53:20.935120+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: cluster 2023-10-09T06:53:20.938794+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:21 smithi155 bash[59495]: cluster 2023-10-09T06:53:21.558686+0000 mgr.y (mgr.25723) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:22.293 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: cluster 2023-10-09T06:53:20.934745+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: audit 2023-10-09T06:53:20.934933+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: audit 2023-10-09T06:53:20.935047+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: cluster 2023-10-09T06:53:20.935120+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: cluster 2023-10-09T06:53:20.938794+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.321 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[77299]: cluster 2023-10-09T06:53:21.558686+0000 mgr.y (mgr.25723) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: cluster 2023-10-09T06:53:20.934745+0000 osd.4 (osd.4) 346 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: audit 2023-10-09T06:53:20.934933+0000 mon.a (mon.0) 1863 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: audit 2023-10-09T06:53:20.935047+0000 mon.a (mon.0) 1864 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: cluster 2023-10-09T06:53:20.935120+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: cluster 2023-10-09T06:53:20.938794+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e394 with expected crc 2023-10-09T06:53:22.322 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:21 smithi125 bash[80002]: cluster 2023-10-09T06:53:21.558686+0000 mgr.y (mgr.25723) 1471 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:23.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:22 smithi155 bash[59495]: cluster 2023-10-09T06:53:21.939315+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:22 smithi155 bash[59495]: cluster 2023-10-09T06:53:21.940355+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:22 smithi155 bash[59495]: cluster 2023-10-09T06:53:21.940731+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:53:23.245 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:22 smithi155 bash[59495]: cluster 2023-10-09T06:53:21.942362+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[77299]: cluster 2023-10-09T06:53:21.939315+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[77299]: cluster 2023-10-09T06:53:21.940355+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[77299]: cluster 2023-10-09T06:53:21.940731+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[77299]: cluster 2023-10-09T06:53:21.942362+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[80002]: cluster 2023-10-09T06:53:21.939315+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[80002]: cluster 2023-10-09T06:53:21.940355+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[80002]: cluster 2023-10-09T06:53:21.940731+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:53:23.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:22 smithi125 bash[80002]: cluster 2023-10-09T06:53:21.942362+0000 osd.4 (osd.4) 347 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:24.330 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:23 smithi155 bash[59495]: cluster 2023-10-09T06:53:22.036037+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:24.330 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:23 smithi155 bash[59495]: cluster 2023-10-09T06:53:23.559252+0000 mgr.y (mgr.25723) 1472 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:24.386 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:23 smithi125 bash[80002]: cluster 2023-10-09T06:53:22.036037+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:24.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:23 smithi125 bash[80002]: cluster 2023-10-09T06:53:23.559252+0000 mgr.y (mgr.25723) 1472 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:24.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:23 smithi125 bash[77299]: cluster 2023-10-09T06:53:22.036037+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e395 with expected crc 2023-10-09T06:53:24.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:23 smithi125 bash[77299]: cluster 2023-10-09T06:53:23.559252+0000 mgr.y (mgr.25723) 1472 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:25.538 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:25.538 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:25 smithi125 bash[77299]: cluster 2023-10-09T06:53:25.560118+0000 mgr.y (mgr.25723) 1473 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:25 smithi125 bash[80002]: cluster 2023-10-09T06:53:25.560118+0000 mgr.y (mgr.25723) 1473 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:25 smithi155 bash[59495]: cluster 2023-10-09T06:53:25.560118+0000 mgr.y (mgr.25723) 1473 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:26.502 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:53:26.652 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:53:26.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:26 smithi155 bash[59495]: audit 2023-10-09T06:53:26.269408+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.125:0/849112929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:26.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:26 smithi155 bash[59495]: audit 2023-10-09T06:53:26.270085+0000 mon.a (mon.0) 1867 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:26.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:26 smithi125 bash[77299]: audit 2023-10-09T06:53:26.269408+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.125:0/849112929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:26.911 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:26 smithi125 bash[77299]: audit 2023-10-09T06:53:26.270085+0000 mon.a (mon.0) 1867 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:26.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:26 smithi125 bash[80002]: audit 2023-10-09T06:53:26.269408+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.125:0/849112929' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:26.911 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:26 smithi125 bash[80002]: audit 2023-10-09T06:53:26.270085+0000 mon.a (mon.0) 1867 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:27.672 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: cluster 2023-10-09T06:53:26.642834+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: audit 2023-10-09T06:53:26.644447+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: cluster 2023-10-09T06:53:26.644549+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: cluster 2023-10-09T06:53:26.654919+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: cluster 2023-10-09T06:53:26.956143+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[77299]: cluster 2023-10-09T06:53:27.560776+0000 mgr.y (mgr.25723) 1474 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:28.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: cluster 2023-10-09T06:53:26.642834+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: audit 2023-10-09T06:53:26.644447+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: cluster 2023-10-09T06:53:26.644549+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:53:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: cluster 2023-10-09T06:53:26.654919+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: cluster 2023-10-09T06:53:26.956143+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:27 smithi125 bash[80002]: cluster 2023-10-09T06:53:27.560776+0000 mgr.y (mgr.25723) 1474 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: cluster 2023-10-09T06:53:26.642834+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: audit 2023-10-09T06:53:26.644447+0000 mon.a (mon.0) 1868 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: cluster 2023-10-09T06:53:26.644549+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: cluster 2023-10-09T06:53:26.654919+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: cluster 2023-10-09T06:53:26.956143+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e396 with expected crc 2023-10-09T06:53:28.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:27 smithi155 bash[59495]: cluster 2023-10-09T06:53:27.560776+0000 mgr.y (mgr.25723) 1474 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:30.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:30 smithi125 bash[77299]: cluster 2023-10-09T06:53:29.561664+0000 mgr.y (mgr.25723) 1475 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:30.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:30 smithi125 bash[80002]: cluster 2023-10-09T06:53:29.561664+0000 mgr.y (mgr.25723) 1475 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:30.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:30 smithi155 bash[59495]: cluster 2023-10-09T06:53:29.561664+0000 mgr.y (mgr.25723) 1475 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:31.284 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:53:31.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:31] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:53:31.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:31 smithi155 bash[59495]: cluster 2023-10-09T06:53:31.562616+0000 mgr.y (mgr.25723) 1476 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:32.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:31 smithi125 bash[77299]: cluster 2023-10-09T06:53:31.562616+0000 mgr.y (mgr.25723) 1476 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:31 smithi125 bash[80002]: cluster 2023-10-09T06:53:31.562616+0000 mgr.y (mgr.25723) 1476 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:32.443 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:33.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:33 smithi125 bash[80002]: cluster 2023-10-09T06:53:33.563109+0000 mgr.y (mgr.25723) 1477 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:33.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:33 smithi125 bash[80002]: audit 2023-10-09T06:53:33.620334+0000 mon.a (mon.0) 1870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:33 smithi125 bash[77299]: cluster 2023-10-09T06:53:33.563109+0000 mgr.y (mgr.25723) 1477 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:33.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:33 smithi125 bash[77299]: audit 2023-10-09T06:53:33.620334+0000 mon.a (mon.0) 1870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:33 smithi155 bash[59495]: cluster 2023-10-09T06:53:33.563109+0000 mgr.y (mgr.25723) 1477 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:33.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:33 smithi155 bash[59495]: audit 2023-10-09T06:53:33.620334+0000 mon.a (mon.0) 1870 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:34.858 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:34.858 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:35.733 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:53:35.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:35 smithi125 bash[77299]: cluster 2023-10-09T06:53:35.563896+0000 mgr.y (mgr.25723) 1478 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:35.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:35 smithi125 bash[80002]: cluster 2023-10-09T06:53:35.563896+0000 mgr.y (mgr.25723) 1478 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:35 smithi155 bash[59495]: cluster 2023-10-09T06:53:35.563896+0000 mgr.y (mgr.25723) 1478 : cluster 0 pgmap v1673: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:36.655 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T06:53:36.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:36 smithi155 bash[59495]: audit 2023-10-09T06:53:36.183633+0000 mon.a (mon.0) 1871 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:36.901 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:36 smithi125 bash[77299]: audit 2023-10-09T06:53:36.183633+0000 mon.a (mon.0) 1871 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:36.901 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:36 smithi125 bash[80002]: audit 2023-10-09T06:53:36.183633+0000 mon.a (mon.0) 1871 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:53:37.634 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:53:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[77299]: cluster 2023-10-09T06:53:36.645896+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[77299]: cluster 2023-10-09T06:53:36.646023+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[77299]: audit 2023-10-09T06:53:36.647412+0000 mon.a (mon.0) 1872 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[77299]: cluster 2023-10-09T06:53:36.647496+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[77299]: cluster 2023-10-09T06:53:37.564580+0000 mgr.y (mgr.25723) 1479 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[80002]: cluster 2023-10-09T06:53:36.645896+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[80002]: cluster 2023-10-09T06:53:36.646023+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[80002]: audit 2023-10-09T06:53:36.647412+0000 mon.a (mon.0) 1872 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:37.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[80002]: cluster 2023-10-09T06:53:36.647496+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:53:37.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:37 smithi125 bash[80002]: cluster 2023-10-09T06:53:37.564580+0000 mgr.y (mgr.25723) 1479 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:37 smithi155 bash[59495]: cluster 2023-10-09T06:53:36.645896+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:37 smithi155 bash[59495]: cluster 2023-10-09T06:53:36.646023+0000 osd.4 (osd.4) 348 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:37 smithi155 bash[59495]: audit 2023-10-09T06:53:36.647412+0000 mon.a (mon.0) 1872 : audit 1 from='client.? 172.21.15.125:0/1845052375' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:53:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:37 smithi155 bash[59495]: cluster 2023-10-09T06:53:36.647496+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:53:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:37 smithi155 bash[59495]: cluster 2023-10-09T06:53:37.564580+0000 mgr.y (mgr.25723) 1479 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:39.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:38 smithi125 bash[77299]: cluster 2023-10-09T06:53:37.784229+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:38 smithi125 bash[80002]: cluster 2023-10-09T06:53:37.784229+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:39.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:38 smithi155 bash[59495]: cluster 2023-10-09T06:53:37.784229+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e397 with expected crc 2023-10-09T06:53:40.508 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:53:40.544 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:40 smithi155 bash[59495]: cluster 2023-10-09T06:53:39.565438+0000 mgr.y (mgr.25723) 1480 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:40.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:40 smithi125 bash[77299]: cluster 2023-10-09T06:53:39.565438+0000 mgr.y (mgr.25723) 1480 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:40.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:40 smithi125 bash[80002]: cluster 2023-10-09T06:53:39.565438+0000 mgr.y (mgr.25723) 1480 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:41.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:41] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:53:41.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:41 smithi155 bash[59495]: cluster 2023-10-09T06:53:41.566292+0000 mgr.y (mgr.25723) 1481 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:41 smithi125 bash[77299]: cluster 2023-10-09T06:53:41.566292+0000 mgr.y (mgr.25723) 1481 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:42.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:41 smithi125 bash[80002]: cluster 2023-10-09T06:53:41.566292+0000 mgr.y (mgr.25723) 1481 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:42.416 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:43.818 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:43.818 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:43.862 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:43 smithi155 bash[59495]: cluster 2023-10-09T06:53:43.567073+0000 mgr.y (mgr.25723) 1482 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:43.879 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:43 smithi125 bash[77299]: cluster 2023-10-09T06:53:43.567073+0000 mgr.y (mgr.25723) 1482 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:43.880 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:43 smithi125 bash[80002]: cluster 2023-10-09T06:53:43.567073+0000 mgr.y (mgr.25723) 1482 : cluster 0 pgmap v1678: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:44.673 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:53:45.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:45 smithi155 bash[59495]: cluster 2023-10-09T06:53:45.567773+0000 mgr.y (mgr.25723) 1483 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:46.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:45 smithi125 bash[77299]: cluster 2023-10-09T06:53:45.567773+0000 mgr.y (mgr.25723) 1483 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:46.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:45 smithi125 bash[80002]: cluster 2023-10-09T06:53:45.567773+0000 mgr.y (mgr.25723) 1483 : cluster 0 pgmap v1679: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:46.655 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:53:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:46 smithi155 bash[59495]: audit 2023-10-09T06:53:46.136144+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.125:0/595586949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:46 smithi155 bash[59495]: audit 2023-10-09T06:53:46.136712+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:47.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:46 smithi125 bash[77299]: audit 2023-10-09T06:53:46.136144+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.125:0/595586949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:47.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:46 smithi125 bash[77299]: audit 2023-10-09T06:53:46.136712+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:47.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:46 smithi125 bash[80002]: audit 2023-10-09T06:53:46.136144+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.125:0/595586949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:47.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:46 smithi125 bash[80002]: audit 2023-10-09T06:53:46.136712+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:47.584 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T06:53:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[77299]: cluster 2023-10-09T06:53:46.646438+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[77299]: cluster 2023-10-09T06:53:46.647322+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[77299]: audit 2023-10-09T06:53:46.647737+0000 mon.a (mon.0) 1875 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[77299]: cluster 2023-10-09T06:53:46.647816+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[77299]: cluster 2023-10-09T06:53:47.568316+0000 mgr.y (mgr.25723) 1484 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[80002]: cluster 2023-10-09T06:53:46.646438+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[80002]: cluster 2023-10-09T06:53:46.647322+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[80002]: audit 2023-10-09T06:53:46.647737+0000 mon.a (mon.0) 1875 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[80002]: cluster 2023-10-09T06:53:46.647816+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:53:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:47 smithi125 bash[80002]: cluster 2023-10-09T06:53:47.568316+0000 mgr.y (mgr.25723) 1484 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:47 smithi155 bash[59495]: cluster 2023-10-09T06:53:46.646438+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:47 smithi155 bash[59495]: cluster 2023-10-09T06:53:46.647322+0000 osd.4 (osd.4) 349 : cluster 3 failed to encode map e398 with expected crc 2023-10-09T06:53:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:47 smithi155 bash[59495]: audit 2023-10-09T06:53:46.647737+0000 mon.a (mon.0) 1875 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:47 smithi155 bash[59495]: cluster 2023-10-09T06:53:46.647816+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:53:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:47 smithi155 bash[59495]: cluster 2023-10-09T06:53:47.568316+0000 mgr.y (mgr.25723) 1484 : cluster 0 pgmap v1681: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:49.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:48 smithi125 bash[77299]: audit 2023-10-09T06:53:48.620198+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:49.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:48 smithi125 bash[80002]: audit 2023-10-09T06:53:48.620198+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:49.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:48 smithi155 bash[59495]: audit 2023-10-09T06:53:48.620198+0000 mon.a (mon.0) 1877 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:53:49.451 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:53:50.625 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:50 smithi155 bash[59495]: cluster 2023-10-09T06:53:49.569207+0000 mgr.y (mgr.25723) 1485 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:50.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:50 smithi125 bash[77299]: cluster 2023-10-09T06:53:49.569207+0000 mgr.y (mgr.25723) 1485 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:50.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:50 smithi125 bash[80002]: cluster 2023-10-09T06:53:49.569207+0000 mgr.y (mgr.25723) 1485 : cluster 0 pgmap v1682: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:51.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:53:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:53:51] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:53:51.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:51 smithi155 bash[59495]: cluster 2023-10-09T06:53:51.570197+0000 mgr.y (mgr.25723) 1486 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:52.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:51 smithi125 bash[77299]: cluster 2023-10-09T06:53:51.570197+0000 mgr.y (mgr.25723) 1486 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:52.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:51 smithi125 bash[80002]: cluster 2023-10-09T06:53:51.570197+0000 mgr.y (mgr.25723) 1486 : cluster 0 pgmap v1683: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:52.358 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:53:52.822 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:53:52.822 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:53:53.664 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:53:53.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:53 smithi125 bash[80002]: cluster 2023-10-09T06:53:53.571041+0000 mgr.y (mgr.25723) 1487 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:53.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:53 smithi125 bash[77299]: cluster 2023-10-09T06:53:53.571041+0000 mgr.y (mgr.25723) 1487 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:53.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:53 smithi155 bash[59495]: cluster 2023-10-09T06:53:53.571041+0000 mgr.y (mgr.25723) 1487 : cluster 0 pgmap v1684: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:55.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:55 smithi125 bash[77299]: cluster 2023-10-09T06:53:55.571949+0000 mgr.y (mgr.25723) 1488 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:55.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:55 smithi125 bash[80002]: cluster 2023-10-09T06:53:55.571949+0000 mgr.y (mgr.25723) 1488 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:55.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:55 smithi155 bash[59495]: cluster 2023-10-09T06:53:55.571949+0000 mgr.y (mgr.25723) 1488 : cluster 0 pgmap v1685: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:56.660 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T06:53:56.905 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:56 smithi125 bash[77299]: audit 2023-10-09T06:53:56.004286+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.125:0/2082079535' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:56.905 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:56 smithi125 bash[77299]: audit 2023-10-09T06:53:56.005002+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:56.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:56 smithi125 bash[80002]: audit 2023-10-09T06:53:56.004286+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.125:0/2082079535' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:56.906 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:56 smithi125 bash[80002]: audit 2023-10-09T06:53:56.005002+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:56 smithi155 bash[59495]: audit 2023-10-09T06:53:56.004286+0000 mon.c (mon.2) 146 : audit 1 from='client.? 172.21.15.125:0/2082079535' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:57.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:56 smithi155 bash[59495]: audit 2023-10-09T06:53:56.005002+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:53:57.647 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:53:58.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[80002]: cluster 2023-10-09T06:53:56.650245+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[80002]: audit 2023-10-09T06:53:56.652965+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[80002]: cluster 2023-10-09T06:53:56.653084+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[80002]: cluster 2023-10-09T06:53:56.763251+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[80002]: cluster 2023-10-09T06:53:57.572613+0000 mgr.y (mgr.25723) 1489 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[77299]: cluster 2023-10-09T06:53:56.650245+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[77299]: audit 2023-10-09T06:53:56.652965+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[77299]: cluster 2023-10-09T06:53:56.653084+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:53:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[77299]: cluster 2023-10-09T06:53:56.763251+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.138 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:53:57 smithi125 bash[77299]: cluster 2023-10-09T06:53:57.572613+0000 mgr.y (mgr.25723) 1489 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:57 smithi155 bash[59495]: cluster 2023-10-09T06:53:56.650245+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:57 smithi155 bash[59495]: audit 2023-10-09T06:53:56.652965+0000 mon.a (mon.0) 1879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:53:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:57 smithi155 bash[59495]: cluster 2023-10-09T06:53:56.653084+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:53:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:57 smithi155 bash[59495]: cluster 2023-10-09T06:53:56.763251+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e399 with expected crc 2023-10-09T06:53:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:53:57 smithi155 bash[59495]: cluster 2023-10-09T06:53:57.572613+0000 mgr.y (mgr.25723) 1489 : cluster 0 pgmap v1687: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:53:58.440 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:00.571 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:00 smithi155 bash[59495]: cluster 2023-10-09T06:53:59.573504+0000 mgr.y (mgr.25723) 1490 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:00.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:00 smithi125 bash[77299]: cluster 2023-10-09T06:53:59.573504+0000 mgr.y (mgr.25723) 1490 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:00.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:00 smithi125 bash[80002]: cluster 2023-10-09T06:53:59.573504+0000 mgr.y (mgr.25723) 1490 : cluster 0 pgmap v1688: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:01.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:01] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:54:01.840 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:01.840 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:01.887 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:01 smithi155 bash[59495]: cluster 2023-10-09T06:54:01.574350+0000 mgr.y (mgr.25723) 1491 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:02.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:01 smithi125 bash[77299]: cluster 2023-10-09T06:54:01.574350+0000 mgr.y (mgr.25723) 1491 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:02.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:01 smithi125 bash[80002]: cluster 2023-10-09T06:54:01.574350+0000 mgr.y (mgr.25723) 1491 : cluster 0 pgmap v1689: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:02.423 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:02.762 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:54:03.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:03 smithi125 bash[77299]: cluster 2023-10-09T06:54:03.574984+0000 mgr.y (mgr.25723) 1492 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:03.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:03 smithi125 bash[77299]: audit 2023-10-09T06:54:03.621114+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:03.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:03 smithi125 bash[80002]: cluster 2023-10-09T06:54:03.574984+0000 mgr.y (mgr.25723) 1492 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:03.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:03 smithi125 bash[80002]: audit 2023-10-09T06:54:03.621114+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:03.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:03 smithi155 bash[59495]: cluster 2023-10-09T06:54:03.574984+0000 mgr.y (mgr.25723) 1492 : cluster 0 pgmap v1690: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:03.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:03 smithi155 bash[59495]: audit 2023-10-09T06:54:03.621114+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:05.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:05 smithi125 bash[77299]: cluster 2023-10-09T06:54:05.576086+0000 mgr.y (mgr.25723) 1493 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:05.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:05 smithi125 bash[80002]: cluster 2023-10-09T06:54:05.576086+0000 mgr.y (mgr.25723) 1493 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:05 smithi155 bash[59495]: cluster 2023-10-09T06:54:05.576086+0000 mgr.y (mgr.25723) 1493 : cluster 0 pgmap v1691: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:06.661 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:54:06.905 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:06 smithi125 bash[77299]: audit 2023-10-09T06:54:06.027961+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:06.905 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:06 smithi125 bash[80002]: audit 2023-10-09T06:54:06.027961+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:07.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:06 smithi155 bash[59495]: audit 2023-10-09T06:54:06.027961+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:07.541 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:07.680 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: audit 2023-10-09T06:54:06.653630+0000 mon.a (mon.0) 1883 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: cluster 2023-10-09T06:54:06.653713+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: cluster 2023-10-09T06:54:06.747883+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: cluster 2023-10-09T06:54:07.166481+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: cluster 2023-10-09T06:54:07.169169+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:07.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:07 smithi155 bash[59495]: cluster 2023-10-09T06:54:07.576779+0000 mgr.y (mgr.25723) 1494 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:08.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: audit 2023-10-09T06:54:06.653630+0000 mon.a (mon.0) 1883 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: cluster 2023-10-09T06:54:06.653713+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: cluster 2023-10-09T06:54:06.747883+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: cluster 2023-10-09T06:54:07.166481+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: cluster 2023-10-09T06:54:07.169169+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[77299]: cluster 2023-10-09T06:54:07.576779+0000 mgr.y (mgr.25723) 1494 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: audit 2023-10-09T06:54:06.653630+0000 mon.a (mon.0) 1883 : audit 1 from='client.? 172.21.15.125:0/7870555' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: cluster 2023-10-09T06:54:06.653713+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: cluster 2023-10-09T06:54:06.747883+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: cluster 2023-10-09T06:54:07.166481+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: cluster 2023-10-09T06:54:07.169169+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:07 smithi125 bash[80002]: cluster 2023-10-09T06:54:07.576779+0000 mgr.y (mgr.25723) 1494 : cluster 0 pgmap v1693: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:08.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:08 smithi155 bash[59495]: cluster 2023-10-09T06:54:07.587824+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:08.954 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:08 smithi155 bash[59495]: cluster 2023-10-09T06:54:07.695887+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:09.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:08 smithi125 bash[77299]: cluster 2023-10-09T06:54:07.587824+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:09.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:08 smithi125 bash[77299]: cluster 2023-10-09T06:54:07.695887+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:09.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:08 smithi125 bash[80002]: cluster 2023-10-09T06:54:07.587824+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:09.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:08 smithi125 bash[80002]: cluster 2023-10-09T06:54:07.695887+0000 osd.4 (osd.4) 350 : cluster 3 failed to encode map e400 with expected crc 2023-10-09T06:54:10.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:10 smithi125 bash[77299]: cluster 2023-10-09T06:54:09.577609+0000 mgr.y (mgr.25723) 1495 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:10.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:10 smithi125 bash[80002]: cluster 2023-10-09T06:54:09.577609+0000 mgr.y (mgr.25723) 1495 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:10.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:10 smithi155 bash[59495]: cluster 2023-10-09T06:54:09.577609+0000 mgr.y (mgr.25723) 1495 : cluster 0 pgmap v1694: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:10.972 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:10.972 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:11.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:11] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:54:11.890 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:54:11.902 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:11 smithi155 bash[59495]: cluster 2023-10-09T06:54:11.578526+0000 mgr.y (mgr.25723) 1496 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:12.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:11 smithi125 bash[77299]: cluster 2023-10-09T06:54:11.578526+0000 mgr.y (mgr.25723) 1496 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:12.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:11 smithi125 bash[80002]: cluster 2023-10-09T06:54:11.578526+0000 mgr.y (mgr.25723) 1496 : cluster 0 pgmap v1695: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:12.447 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:13.944 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:13 smithi125 bash[77299]: cluster 2023-10-09T06:54:13.579002+0000 mgr.y (mgr.25723) 1497 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:13.944 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:13 smithi125 bash[80002]: cluster 2023-10-09T06:54:13.579002+0000 mgr.y (mgr.25723) 1497 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:14.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:13 smithi155 bash[59495]: cluster 2023-10-09T06:54:13.579002+0000 mgr.y (mgr.25723) 1497 : cluster 0 pgmap v1696: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:16.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:15 smithi125 bash[77299]: cluster 2023-10-09T06:54:15.579476+0000 mgr.y (mgr.25723) 1498 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:16.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:15 smithi125 bash[80002]: cluster 2023-10-09T06:54:15.579476+0000 mgr.y (mgr.25723) 1498 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:16.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:15 smithi155 bash[59495]: cluster 2023-10-09T06:54:15.579476+0000 mgr.y (mgr.25723) 1498 : cluster 0 pgmap v1697: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:16.665 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T06:54:16.669 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:17.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:16 smithi125 bash[80002]: audit 2023-10-09T06:54:16.235851+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.125:0/1961447444' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:16 smithi125 bash[80002]: audit 2023-10-09T06:54:16.236573+0000 mon.a (mon.0) 1885 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:16 smithi125 bash[77299]: audit 2023-10-09T06:54:16.235851+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.125:0/1961447444' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:16 smithi125 bash[77299]: audit 2023-10-09T06:54:16.236573+0000 mon.a (mon.0) 1885 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:16 smithi155 bash[59495]: audit 2023-10-09T06:54:16.235851+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.125:0/1961447444' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:16 smithi155 bash[59495]: audit 2023-10-09T06:54:16.236573+0000 mon.a (mon.0) 1885 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:54:17.644 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:54:17.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:17 smithi155 bash[59495]: audit 2023-10-09T06:54:16.653736+0000 mon.a (mon.0) 1886 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:17.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:17 smithi155 bash[59495]: cluster 2023-10-09T06:54:16.653793+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:54:17.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:17 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.068479+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:17.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:17 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.071358+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:17.986 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:17 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.580092+0000 mgr.y (mgr.25723) 1499 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:18.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[77299]: audit 2023-10-09T06:54:16.653736+0000 mon.a (mon.0) 1886 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[77299]: cluster 2023-10-09T06:54:16.653793+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.068479+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.071358+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.580092+0000 mgr.y (mgr.25723) 1499 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[80002]: audit 2023-10-09T06:54:16.653736+0000 mon.a (mon.0) 1886 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[80002]: cluster 2023-10-09T06:54:16.653793+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:54:18.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.068479+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.071358+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:18.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:17 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.580092+0000 mgr.y (mgr.25723) 1499 : cluster 0 pgmap v1699: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.462081+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.730649+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[77299]: cluster 2023-10-09T06:54:17.855153+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[77299]: audit 2023-10-09T06:54:18.621022+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.462081+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.730649+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[80002]: cluster 2023-10-09T06:54:17.855153+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:18 smithi125 bash[80002]: audit 2023-10-09T06:54:18.621022+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:18 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.462081+0000 osd.4 (osd.4) 351 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:18 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.730649+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:18 smithi155 bash[59495]: cluster 2023-10-09T06:54:17.855153+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e401 with expected crc 2023-10-09T06:54:19.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:18 smithi155 bash[59495]: audit 2023-10-09T06:54:18.621022+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:20.237 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:20.238 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:20 smithi125 bash[77299]: cluster 2023-10-09T06:54:19.581044+0000 mgr.y (mgr.25723) 1500 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:20.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:20 smithi125 bash[77299]: audit 2023-10-09T06:54:20.013279+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:54:20.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:20 smithi125 bash[80002]: cluster 2023-10-09T06:54:19.581044+0000 mgr.y (mgr.25723) 1500 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:20.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:20 smithi125 bash[80002]: audit 2023-10-09T06:54:20.013279+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:54:20.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:20 smithi155 bash[59495]: cluster 2023-10-09T06:54:19.581044+0000 mgr.y (mgr.25723) 1500 : cluster 0 pgmap v1700: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:20.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:20 smithi155 bash[59495]: audit 2023-10-09T06:54:20.013279+0000 mon.a (mon.0) 1889 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:54:21.167 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:54:21.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:21 smithi125 bash[77299]: audit 2023-10-09T06:54:20.391326+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:21.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:21 smithi125 bash[77299]: audit 2023-10-09T06:54:20.399728+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:21.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:21] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:54:21.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:21 smithi125 bash[80002]: audit 2023-10-09T06:54:20.391326+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:21.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:21 smithi125 bash[80002]: audit 2023-10-09T06:54:20.399728+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:21 smithi155 bash[59495]: audit 2023-10-09T06:54:20.391326+0000 mon.a (mon.0) 1890 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:21.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:21 smithi155 bash[59495]: audit 2023-10-09T06:54:20.399728+0000 mon.a (mon.0) 1891 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:22.426 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:22.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:22 smithi125 bash[77299]: cluster 2023-10-09T06:54:21.581891+0000 mgr.y (mgr.25723) 1501 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:22.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:22 smithi125 bash[80002]: cluster 2023-10-09T06:54:21.581891+0000 mgr.y (mgr.25723) 1501 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:22.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:22 smithi155 bash[59495]: cluster 2023-10-09T06:54:21.581891+0000 mgr.y (mgr.25723) 1501 : cluster 0 pgmap v1701: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:23.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:23 smithi125 bash[77299]: cluster 2023-10-09T06:54:23.582680+0000 mgr.y (mgr.25723) 1502 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:23.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:23 smithi125 bash[80002]: cluster 2023-10-09T06:54:23.582680+0000 mgr.y (mgr.25723) 1502 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:24.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:23 smithi155 bash[59495]: cluster 2023-10-09T06:54:23.582680+0000 mgr.y (mgr.25723) 1502 : cluster 0 pgmap v1702: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:25.932 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:25.962 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:25 smithi155 bash[59495]: cluster 2023-10-09T06:54:25.583509+0000 mgr.y (mgr.25723) 1503 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:26.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:25 smithi125 bash[77299]: cluster 2023-10-09T06:54:25.583509+0000 mgr.y (mgr.25723) 1503 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:26.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:25 smithi125 bash[80002]: cluster 2023-10-09T06:54:25.583509+0000 mgr.y (mgr.25723) 1503 : cluster 0 pgmap v1703: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:26.663 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:54:27.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:26 smithi155 bash[59495]: audit 2023-10-09T06:54:26.174371+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:27.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:26 smithi155 bash[59495]: audit 2023-10-09T06:54:26.289792+0000 mon.a (mon.0) 1893 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.079 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:26 smithi155 bash[59495]: audit 2023-10-09T06:54:26.296149+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:26 smithi155 bash[59495]: audit 2023-10-09T06:54:26.412143+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.080 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:26 smithi155 bash[59495]: audit 2023-10-09T06:54:26.419057+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[77299]: audit 2023-10-09T06:54:26.174371+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:27.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[77299]: audit 2023-10-09T06:54:26.289792+0000 mon.a (mon.0) 1893 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[77299]: audit 2023-10-09T06:54:26.296149+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[77299]: audit 2023-10-09T06:54:26.412143+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[77299]: audit 2023-10-09T06:54:26.419057+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[80002]: audit 2023-10-09T06:54:26.174371+0000 mon.a (mon.0) 1892 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:27.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[80002]: audit 2023-10-09T06:54:26.289792+0000 mon.a (mon.0) 1893 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[80002]: audit 2023-10-09T06:54:26.296149+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[80002]: audit 2023-10-09T06:54:26.412143+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:26 smithi125 bash[80002]: audit 2023-10-09T06:54:26.419057+0000 mon.a (mon.0) 1896 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.658 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: audit 2023-10-09T06:54:26.656290+0000 mon.a (mon.0) 1897 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: cluster 2023-10-09T06:54:26.656377+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: audit 2023-10-09T06:54:26.893314+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: audit 2023-10-09T06:54:26.894631+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: audit 2023-10-09T06:54:26.900778+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: cluster 2023-10-09T06:54:26.970664+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:27.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: cluster 2023-10-09T06:54:26.971716+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:27.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:27 smithi155 bash[59495]: cluster 2023-10-09T06:54:27.584097+0000 mgr.y (mgr.25723) 1504 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:28.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: audit 2023-10-09T06:54:26.656290+0000 mon.a (mon.0) 1897 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: cluster 2023-10-09T06:54:26.656377+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: audit 2023-10-09T06:54:26.893314+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: audit 2023-10-09T06:54:26.894631+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: audit 2023-10-09T06:54:26.900778+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: cluster 2023-10-09T06:54:26.970664+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: cluster 2023-10-09T06:54:26.971716+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[77299]: cluster 2023-10-09T06:54:27.584097+0000 mgr.y (mgr.25723) 1504 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:28.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: audit 2023-10-09T06:54:26.656290+0000 mon.a (mon.0) 1897 : audit 1 from='client.? 172.21.15.125:0/1361911341' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: cluster 2023-10-09T06:54:26.656377+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: audit 2023-10-09T06:54:26.893314+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: audit 2023-10-09T06:54:26.894631+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: audit 2023-10-09T06:54:26.900778+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: cluster 2023-10-09T06:54:26.970664+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: cluster 2023-10-09T06:54:26.971716+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:28.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:27 smithi125 bash[80002]: cluster 2023-10-09T06:54:27.584097+0000 mgr.y (mgr.25723) 1504 : cluster 0 pgmap v1705: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:29.202 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:28 smithi155 bash[59495]: cluster 2023-10-09T06:54:27.490483+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:29.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:28 smithi125 bash[77299]: cluster 2023-10-09T06:54:27.490483+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:29.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:28 smithi125 bash[80002]: cluster 2023-10-09T06:54:27.490483+0000 osd.4 (osd.4) 352 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:29.457 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:29.457 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:29 smithi155 bash[59495]: cluster 2023-10-09T06:54:28.503889+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:29 smithi155 bash[59495]: cluster 2023-10-09T06:54:28.503916+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:29 smithi155 bash[59495]: cluster 2023-10-09T06:54:29.584952+0000 mgr.y (mgr.25723) 1505 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:30.303 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:54:30.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[77299]: cluster 2023-10-09T06:54:28.503889+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[77299]: cluster 2023-10-09T06:54:28.503916+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.387 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[77299]: cluster 2023-10-09T06:54:29.584952+0000 mgr.y (mgr.25723) 1505 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:30.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[80002]: cluster 2023-10-09T06:54:28.503889+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[80002]: cluster 2023-10-09T06:54:28.503916+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e402 with expected crc 2023-10-09T06:54:30.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:29 smithi125 bash[80002]: cluster 2023-10-09T06:54:29.584952+0000 mgr.y (mgr.25723) 1505 : cluster 0 pgmap v1706: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:31.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:31] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:54:32.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:31 smithi125 bash[77299]: cluster 2023-10-09T06:54:31.585918+0000 mgr.y (mgr.25723) 1506 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:32.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:31 smithi125 bash[80002]: cluster 2023-10-09T06:54:31.585918+0000 mgr.y (mgr.25723) 1506 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:32.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:31 smithi155 bash[59495]: cluster 2023-10-09T06:54:31.585918+0000 mgr.y (mgr.25723) 1506 : cluster 0 pgmap v1707: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:32.429 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:34.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:33 smithi125 bash[80002]: cluster 2023-10-09T06:54:33.586656+0000 mgr.y (mgr.25723) 1507 : cluster 0 pgmap v1708: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:34.041 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:33 smithi125 bash[80002]: audit 2023-10-09T06:54:33.621404+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:34.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:33 smithi125 bash[77299]: cluster 2023-10-09T06:54:33.586656+0000 mgr.y (mgr.25723) 1507 : cluster 0 pgmap v1708: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:34.041 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:33 smithi125 bash[77299]: audit 2023-10-09T06:54:33.621404+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:34.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:33 smithi155 bash[59495]: cluster 2023-10-09T06:54:33.586656+0000 mgr.y (mgr.25723) 1507 : cluster 0 pgmap v1708: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:34.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:33 smithi155 bash[59495]: audit 2023-10-09T06:54:33.621404+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:35.076 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:36.136 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:35 smithi125 bash[80002]: cluster 2023-10-09T06:54:35.587164+0000 mgr.y (mgr.25723) 1508 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:36.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:35 smithi125 bash[77299]: cluster 2023-10-09T06:54:35.587164+0000 mgr.y (mgr.25723) 1508 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:36.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:35 smithi155 bash[59495]: cluster 2023-10-09T06:54:35.587164+0000 mgr.y (mgr.25723) 1508 : cluster 0 pgmap v1709: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:36.674 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:54:37.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:36 smithi125 bash[77299]: audit 2023-10-09T06:54:36.230919+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:37.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:36 smithi125 bash[80002]: audit 2023-10-09T06:54:36.230919+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:37.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:36 smithi155 bash[59495]: audit 2023-10-09T06:54:36.230919+0000 mon.a (mon.0) 1903 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:54:37.548 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:54:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[77299]: cluster 2023-10-09T06:54:36.664026+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[77299]: audit 2023-10-09T06:54:36.665554+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[77299]: cluster 2023-10-09T06:54:36.665640+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.587628+0000 mgr.y (mgr.25723) 1509 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[80002]: cluster 2023-10-09T06:54:36.664026+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[80002]: audit 2023-10-09T06:54:36.665554+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[80002]: cluster 2023-10-09T06:54:36.665640+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:54:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:37 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.587628+0000 mgr.y (mgr.25723) 1509 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:37 smithi155 bash[59495]: cluster 2023-10-09T06:54:36.664026+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e403 with expected crc 2023-10-09T06:54:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:37 smithi155 bash[59495]: audit 2023-10-09T06:54:36.665554+0000 mon.a (mon.0) 1904 : audit 1 from='client.? 172.21.15.125:0/2606680644' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:54:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:37 smithi155 bash[59495]: cluster 2023-10-09T06:54:36.665640+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:54:38.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:37 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.587628+0000 mgr.y (mgr.25723) 1509 : cluster 0 pgmap v1711: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:38.491 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:38.491 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:39.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.671381+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.672315+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.673712+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.674815+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[77299]: cluster 2023-10-09T06:54:37.858849+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.671381+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.672315+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.673712+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:54:39.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.674815+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.138 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:38 smithi125 bash[80002]: cluster 2023-10-09T06:54:37.858849+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:38 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.671381+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:38 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.672315+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:38 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.673712+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:54:39.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:38 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.674815+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:38 smithi155 bash[59495]: cluster 2023-10-09T06:54:37.858849+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e404 with expected crc 2023-10-09T06:54:39.403 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:54:40.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:40 smithi125 bash[77299]: cluster 2023-10-09T06:54:39.588556+0000 mgr.y (mgr.25723) 1510 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:40.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:40 smithi125 bash[80002]: cluster 2023-10-09T06:54:39.588556+0000 mgr.y (mgr.25723) 1510 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:40.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:40 smithi155 bash[59495]: cluster 2023-10-09T06:54:39.588556+0000 mgr.y (mgr.25723) 1510 : cluster 0 pgmap v1713: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:41.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:41] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:54:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:41 smithi125 bash[77299]: cluster 2023-10-09T06:54:41.589486+0000 mgr.y (mgr.25723) 1511 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:42.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:41 smithi125 bash[80002]: cluster 2023-10-09T06:54:41.589486+0000 mgr.y (mgr.25723) 1511 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:41 smithi155 bash[59495]: cluster 2023-10-09T06:54:41.589486+0000 mgr.y (mgr.25723) 1511 : cluster 0 pgmap v1714: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:42.325 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:43.865 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:43 smithi125 bash[77299]: cluster 2023-10-09T06:54:43.590212+0000 mgr.y (mgr.25723) 1512 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:43.865 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:43 smithi125 bash[80002]: cluster 2023-10-09T06:54:43.590212+0000 mgr.y (mgr.25723) 1512 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:44.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:43 smithi155 bash[59495]: cluster 2023-10-09T06:54:43.590212+0000 mgr.y (mgr.25723) 1512 : cluster 0 pgmap v1715: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:44.180 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:45.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:45 smithi125 bash[77299]: cluster 2023-10-09T06:54:45.591175+0000 mgr.y (mgr.25723) 1513 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:45.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:45 smithi125 bash[80002]: cluster 2023-10-09T06:54:45.591175+0000 mgr.y (mgr.25723) 1513 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:46.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:45 smithi155 bash[59495]: cluster 2023-10-09T06:54:45.591175+0000 mgr.y (mgr.25723) 1513 : cluster 0 pgmap v1716: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:46.670 INFO:teuthology.orchestra.run.smithi125.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:54:46.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:46 smithi155 bash[59495]: audit 2023-10-09T06:54:46.085621+0000 mon.a (mon.0) 1907 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:46.914 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:46 smithi125 bash[77299]: audit 2023-10-09T06:54:46.085621+0000 mon.a (mon.0) 1907 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:46.914 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:46 smithi125 bash[80002]: audit 2023-10-09T06:54:46.085621+0000 mon.a (mon.0) 1907 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:47.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T06:54:47.626 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:47.626 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:47.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[77299]: cluster 2023-10-09T06:54:46.661543+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[77299]: audit 2023-10-09T06:54:46.662825+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[77299]: cluster 2023-10-09T06:54:46.662904+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[77299]: cluster 2023-10-09T06:54:47.176713+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[77299]: cluster 2023-10-09T06:54:47.591716+0000 mgr.y (mgr.25723) 1514 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[80002]: cluster 2023-10-09T06:54:46.661543+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[80002]: audit 2023-10-09T06:54:46.662825+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[80002]: cluster 2023-10-09T06:54:46.662904+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:54:47.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[80002]: cluster 2023-10-09T06:54:47.176713+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:47.888 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:47 smithi125 bash[80002]: cluster 2023-10-09T06:54:47.591716+0000 mgr.y (mgr.25723) 1514 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:47 smithi155 bash[59495]: cluster 2023-10-09T06:54:46.661543+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:47 smithi155 bash[59495]: audit 2023-10-09T06:54:46.662825+0000 mon.a (mon.0) 1908 : audit 1 from='client.? 172.21.15.125:0/2679045398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:47 smithi155 bash[59495]: cluster 2023-10-09T06:54:46.662904+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:54:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:47 smithi155 bash[59495]: cluster 2023-10-09T06:54:47.176713+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:48.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:47 smithi155 bash[59495]: cluster 2023-10-09T06:54:47.591716+0000 mgr.y (mgr.25723) 1514 : cluster 0 pgmap v1718: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:48.493 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:54:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:48 smithi155 bash[59495]: cluster 2023-10-09T06:54:47.449799+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:48 smithi155 bash[59495]: cluster 2023-10-09T06:54:47.449872+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:48.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:48 smithi155 bash[59495]: audit 2023-10-09T06:54:48.621926+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:49.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[77299]: cluster 2023-10-09T06:54:47.449799+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:49.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[77299]: cluster 2023-10-09T06:54:47.449872+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:49.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[77299]: audit 2023-10-09T06:54:48.621926+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:49.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[80002]: cluster 2023-10-09T06:54:47.449799+0000 osd.4 (osd.4) 353 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:49.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[80002]: cluster 2023-10-09T06:54:47.449872+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e405 with expected crc 2023-10-09T06:54:49.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:48 smithi125 bash[80002]: audit 2023-10-09T06:54:48.621926+0000 mon.a (mon.0) 1910 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:54:50.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:50 smithi125 bash[77299]: cluster 2023-10-09T06:54:49.592576+0000 mgr.y (mgr.25723) 1515 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:50.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:50 smithi125 bash[80002]: cluster 2023-10-09T06:54:49.592576+0000 mgr.y (mgr.25723) 1515 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:50.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:50 smithi155 bash[59495]: cluster 2023-10-09T06:54:49.592576+0000 mgr.y (mgr.25723) 1515 : cluster 0 pgmap v1719: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:51.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:54:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:54:51] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:54:52.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:51 smithi125 bash[77299]: cluster 2023-10-09T06:54:51.593444+0000 mgr.y (mgr.25723) 1516 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:52.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:51 smithi125 bash[80002]: cluster 2023-10-09T06:54:51.593444+0000 mgr.y (mgr.25723) 1516 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:52.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:51 smithi155 bash[59495]: cluster 2023-10-09T06:54:51.593444+0000 mgr.y (mgr.25723) 1516 : cluster 0 pgmap v1720: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:52.400 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:54:53.272 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:54:54.054 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:53 smithi125 bash[77299]: cluster 2023-10-09T06:54:53.594170+0000 mgr.y (mgr.25723) 1517 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:54.054 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:53 smithi125 bash[80002]: cluster 2023-10-09T06:54:53.594170+0000 mgr.y (mgr.25723) 1517 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:54.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:53 smithi155 bash[59495]: cluster 2023-10-09T06:54:53.594170+0000 mgr.y (mgr.25723) 1517 : cluster 0 pgmap v1721: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:56.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:55 smithi125 bash[77299]: cluster 2023-10-09T06:54:55.595090+0000 mgr.y (mgr.25723) 1518 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:56.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:55 smithi125 bash[80002]: cluster 2023-10-09T06:54:55.595090+0000 mgr.y (mgr.25723) 1518 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:56.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:55 smithi155 bash[59495]: cluster 2023-10-09T06:54:55.595090+0000 mgr.y (mgr.25723) 1518 : cluster 0 pgmap v1722: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:56.677 INFO:teuthology.orchestra.run.smithi125.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T06:54:56.912 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:56 smithi125 bash[77299]: audit 2023-10-09T06:54:56.266645+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:56.913 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:56 smithi125 bash[80002]: audit 2023-10-09T06:54:56.266645+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:56.924 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:54:56.924 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:54:56.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:56 smithi155 bash[59495]: audit 2023-10-09T06:54:56.266645+0000 mon.a (mon.0) 1911 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:54:57.648 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T06:54:57.648 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T06:54:57.648 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T06:54:57.844 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:54:58.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[77299]: cluster 2023-10-09T06:54:56.668620+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[77299]: audit 2023-10-09T06:54:56.669976+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[77299]: cluster 2023-10-09T06:54:56.670084+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[77299]: cluster 2023-10-09T06:54:57.595758+0000 mgr.y (mgr.25723) 1519 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[80002]: cluster 2023-10-09T06:54:56.668620+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[80002]: audit 2023-10-09T06:54:56.669976+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[80002]: cluster 2023-10-09T06:54:56.670084+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:54:58.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:57 smithi125 bash[80002]: cluster 2023-10-09T06:54:57.595758+0000 mgr.y (mgr.25723) 1519 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:57 smithi155 bash[59495]: cluster 2023-10-09T06:54:56.668620+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:58.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:57 smithi155 bash[59495]: audit 2023-10-09T06:54:56.669976+0000 mon.a (mon.0) 1912 : audit 1 from='client.? 172.21.15.125:0/1173764719' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:54:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:57 smithi155 bash[59495]: cluster 2023-10-09T06:54:56.670084+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:54:58.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:57 smithi155 bash[59495]: cluster 2023-10-09T06:54:57.595758+0000 mgr.y (mgr.25723) 1519 : cluster 0 pgmap v1724: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:54:59.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:58 smithi125 bash[77299]: cluster 2023-10-09T06:54:57.350463+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:59.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:58 smithi125 bash[77299]: cluster 2023-10-09T06:54:57.913792+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:58 smithi125 bash[80002]: cluster 2023-10-09T06:54:57.350463+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:59.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:58 smithi125 bash[80002]: cluster 2023-10-09T06:54:57.913792+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:59.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:58 smithi155 bash[59495]: cluster 2023-10-09T06:54:57.350463+0000 osd.4 (osd.4) 354 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:54:59.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:58 smithi155 bash[59495]: cluster 2023-10-09T06:54:57.913792+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:55:00.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:59 smithi125 bash[77299]: cluster 2023-10-09T06:54:58.224079+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:55:00.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:54:59 smithi125 bash[77299]: cluster 2023-10-09T06:54:59.596696+0000 mgr.y (mgr.25723) 1520 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:00.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:59 smithi125 bash[80002]: cluster 2023-10-09T06:54:58.224079+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:55:00.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:54:59 smithi125 bash[80002]: cluster 2023-10-09T06:54:59.596696+0000 mgr.y (mgr.25723) 1520 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:59 smithi155 bash[59495]: cluster 2023-10-09T06:54:58.224079+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e406 with expected crc 2023-10-09T06:55:00.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:54:59 smithi155 bash[59495]: cluster 2023-10-09T06:54:59.596696+0000 mgr.y (mgr.25723) 1520 : cluster 0 pgmap v1725: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:01.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:01] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:55:02.618 INFO:teuthology.orchestra.run.smithi155.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:55:03.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:02 smithi125 bash[77299]: cluster 2023-10-09T06:55:01.597614+0000 mgr.y (mgr.25723) 1521 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:03.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:02 smithi125 bash[80002]: cluster 2023-10-09T06:55:01.597614+0000 mgr.y (mgr.25723) 1521 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:03.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:02 smithi155 bash[59495]: cluster 2023-10-09T06:55:01.597614+0000 mgr.y (mgr.25723) 1521 : cluster 0 pgmap v1726: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:03.974 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:03 smithi155 bash[59495]: audit 2023-10-09T06:55:03.622241+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:04.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:03 smithi125 bash[77299]: audit 2023-10-09T06:55:03.622241+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:04.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:03 smithi125 bash[80002]: audit 2023-10-09T06:55:03.622241+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:05.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:05 smithi125 bash[77299]: cluster 2023-10-09T06:55:03.598350+0000 mgr.y (mgr.25723) 1522 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:05.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:05 smithi125 bash[80002]: cluster 2023-10-09T06:55:03.598350+0000 mgr.y (mgr.25723) 1522 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:05.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:05 smithi155 bash[59495]: cluster 2023-10-09T06:55:03.598350+0000 mgr.y (mgr.25723) 1522 : cluster 0 pgmap v1727: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:05.993 INFO:teuthology.orchestra.run.smithi155.stdout:ERROR: (22) Invalid argument 2023-10-09T06:55:05.994 INFO:teuthology.orchestra.run.smithi155.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:55:06.902 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T06:55:06.903 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:55:06.903 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:55:07.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:07 smithi125 bash[77299]: cluster 2023-10-09T06:55:05.599227+0000 mgr.y (mgr.25723) 1523 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:07.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:07 smithi125 bash[80002]: cluster 2023-10-09T06:55:05.599227+0000 mgr.y (mgr.25723) 1523 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:07.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:07 smithi155 bash[59495]: cluster 2023-10-09T06:55:05.599227+0000 mgr.y (mgr.25723) 1523 : cluster 0 pgmap v1728: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:09.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:09 smithi125 bash[77299]: cluster 2023-10-09T06:55:07.600076+0000 mgr.y (mgr.25723) 1524 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:09.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:09 smithi125 bash[80002]: cluster 2023-10-09T06:55:07.600076+0000 mgr.y (mgr.25723) 1524 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:09.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:09 smithi155 bash[59495]: cluster 2023-10-09T06:55:07.600076+0000 mgr.y (mgr.25723) 1524 : cluster 0 pgmap v1729: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:11.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:11 smithi125 bash[77299]: cluster 2023-10-09T06:55:09.600942+0000 mgr.y (mgr.25723) 1525 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:11.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:11] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:55:11.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:11 smithi125 bash[80002]: cluster 2023-10-09T06:55:09.600942+0000 mgr.y (mgr.25723) 1525 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:11.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:11 smithi155 bash[59495]: cluster 2023-10-09T06:55:09.600942+0000 mgr.y (mgr.25723) 1525 : cluster 0 pgmap v1730: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:11.676 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:13.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:13 smithi125 bash[77299]: cluster 2023-10-09T06:55:11.601794+0000 mgr.y (mgr.25723) 1526 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:13.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:13 smithi125 bash[80002]: cluster 2023-10-09T06:55:11.601794+0000 mgr.y (mgr.25723) 1526 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:13.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:13 smithi155 bash[59495]: cluster 2023-10-09T06:55:11.601794+0000 mgr.y (mgr.25723) 1526 : cluster 0 pgmap v1731: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:15.438 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:55:15.438 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":406,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:54:56.654808+0000","last_up_change":"2023-10-09T06:14:41.646246+0000","last_in_change":"2023-10-09T06:51:52.615770+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"349","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":349,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":388,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6803","nonce":155836221}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6805","nonce":155836221}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6809","nonce":155836221}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6807","nonce":155836221}]},"public_addr":"172.21.15.125:6803/155836221","cluster_addr":"172.21.15.125:6805/155836221","heartbeat_back_addr":"172.21.15.125:6809/155836221","heartbeat_front_addr":"172.21.15.125:6807/155836221","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":390,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6811","nonce":3444691262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6813","nonce":3444691262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6817","nonce":3444691262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6815","nonce":3444691262}]},"public_addr":"172.21.15.125:6811/3444691262","cluster_addr":"172.21.15.125:6813/3444691262","heartbeat_back_addr":"172.21.15.125:6817/3444691262","heartbeat_front_addr":"172.21.15.125:6815/3444691262","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":384,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6819","nonce":1172219518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6821","nonce":1172219518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6825","nonce":1172219518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6823","nonce":1172219518}]},"public_addr":"172.21.15.125:6819/1172219518","cluster_addr":"172.21.15.125:6821/1172219518","heartbeat_back_addr":"172.21.15.125:6825/1172219518","heartbeat_front_addr":"172.21.15.125:6823/1172219518","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":394,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6827","nonce":2373950101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6829","nonce":2373950101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6833","nonce":2373950101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6831","nonce":2373950101}]},"public_addr":"172.21.15.125:6827/2373950101","cluster_addr":"172.21.15.125:6829/2373950101","heartbeat_back_addr":"172.21.15.125:6833/2373950101","heartbeat_front_addr":"172.21.15.125:6831/2373950101","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":394,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":387,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":388,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.4","mappings":[{"from":3,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":3,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.155:6834/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/2757914842":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1505504676":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.155:0/3959201196":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/928092871":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/457536542":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/4007395716":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/2669712956":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6800/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/4000145196":"2023-10-10T06:08:18.401578+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/4016621680":"2023-10-10T06:05:34.904573+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.155:0/497767166":"2023-10-10T06:07:14.674105+0000","172.21.15.155:6835/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/3175453899":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/4031419935":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6801/105318502":"2023-10-10T06:08:18.401578+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:55:15.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:15 smithi125 bash[77299]: cluster 2023-10-09T06:55:13.602514+0000 mgr.y (mgr.25723) 1527 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:15.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:15 smithi125 bash[80002]: cluster 2023-10-09T06:55:13.602514+0000 mgr.y (mgr.25723) 1527 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:15.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:15 smithi155 bash[59495]: cluster 2023-10-09T06:55:13.602514+0000 mgr.y (mgr.25723) 1527 : cluster 0 pgmap v1732: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:16.418 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:55:16.418 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:55:16.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:16 smithi125 bash[77299]: audit 2023-10-09T06:55:15.432558+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.125:0/3522921673' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:16.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:16 smithi125 bash[80002]: audit 2023-10-09T06:55:15.432558+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.125:0/3522921673' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:16.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:16 smithi155 bash[59495]: audit 2023-10-09T06:55:15.432558+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.125:0/3522921673' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:17.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:17 smithi125 bash[77299]: cluster 2023-10-09T06:55:15.603075+0000 mgr.y (mgr.25723) 1528 : cluster 0 pgmap v1733: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:17.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:17 smithi125 bash[80002]: cluster 2023-10-09T06:55:15.603075+0000 mgr.y (mgr.25723) 1528 : cluster 0 pgmap v1733: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:17.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:17 smithi155 bash[59495]: cluster 2023-10-09T06:55:15.603075+0000 mgr.y (mgr.25723) 1528 : cluster 0 pgmap v1733: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:19.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:19 smithi125 bash[77299]: cluster 2023-10-09T06:55:17.603789+0000 mgr.y (mgr.25723) 1529 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:19.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:19 smithi125 bash[77299]: audit 2023-10-09T06:55:18.622790+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:19 smithi125 bash[80002]: cluster 2023-10-09T06:55:17.603789+0000 mgr.y (mgr.25723) 1529 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:19.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:19 smithi125 bash[80002]: audit 2023-10-09T06:55:18.622790+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:19 smithi155 bash[59495]: cluster 2023-10-09T06:55:17.603789+0000 mgr.y (mgr.25723) 1529 : cluster 0 pgmap v1734: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:19.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:19 smithi155 bash[59495]: audit 2023-10-09T06:55:18.622790+0000 mon.a (mon.0) 1915 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:21.198 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:21.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:21 smithi125 bash[77299]: cluster 2023-10-09T06:55:19.604735+0000 mgr.y (mgr.25723) 1530 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:21.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:21] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:55:21.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:21 smithi125 bash[80002]: cluster 2023-10-09T06:55:19.604735+0000 mgr.y (mgr.25723) 1530 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:21.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:21 smithi155 bash[59495]: cluster 2023-10-09T06:55:19.604735+0000 mgr.y (mgr.25723) 1530 : cluster 0 pgmap v1735: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:23.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:23 smithi125 bash[77299]: cluster 2023-10-09T06:55:21.605620+0000 mgr.y (mgr.25723) 1531 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:23.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:23 smithi125 bash[80002]: cluster 2023-10-09T06:55:21.605620+0000 mgr.y (mgr.25723) 1531 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:23.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:23 smithi155 bash[59495]: cluster 2023-10-09T06:55:21.605620+0000 mgr.y (mgr.25723) 1531 : cluster 0 pgmap v1736: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:25.252 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:55:25.252 INFO:teuthology.orchestra.run.smithi125.stdout:{"epoch":406,"fsid":"95d0291c-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:50:59.273052+0000","modified":"2023-10-09T06:54:56.654808+0000","last_up_change":"2023-10-09T06:14:41.646246+0000","last_in_change":"2023-10-09T06:51:52.615770+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:55:15.199759+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:59:51.326050+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"349","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":349,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T06:01:49.244429+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T06:01:50.064272+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f01d5c9b-8b14-4090-a4f3-7203497e2ebe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":84,"up_from":88,"up_thru":388,"down_at":85,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6802","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6803","nonce":155836221}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6804","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6805","nonce":155836221}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6808","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6809","nonce":155836221}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6806","nonce":155836221},{"type":"v1","addr":"172.21.15.125:6807","nonce":155836221}]},"public_addr":"172.21.15.125:6803/155836221","cluster_addr":"172.21.15.125:6805/155836221","heartbeat_back_addr":"172.21.15.125:6809/155836221","heartbeat_front_addr":"172.21.15.125:6807/155836221","state":["exists","up"]},{"osd":1,"uuid":"f3b751bb-a1ce-4054-993d-edd9b0632cc2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":89,"up_from":93,"up_thru":390,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6810","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6811","nonce":3444691262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6812","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6813","nonce":3444691262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6816","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6817","nonce":3444691262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6814","nonce":3444691262},{"type":"v1","addr":"172.21.15.125:6815","nonce":3444691262}]},"public_addr":"172.21.15.125:6811/3444691262","cluster_addr":"172.21.15.125:6813/3444691262","heartbeat_back_addr":"172.21.15.125:6817/3444691262","heartbeat_front_addr":"172.21.15.125:6815/3444691262","state":["exists","up"]},{"osd":2,"uuid":"cc437638-50e5-4487-9372-5300b0e63955","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":94,"up_from":98,"up_thru":384,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6818","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6819","nonce":1172219518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6820","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6821","nonce":1172219518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6824","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6825","nonce":1172219518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6822","nonce":1172219518},{"type":"v1","addr":"172.21.15.125:6823","nonce":1172219518}]},"public_addr":"172.21.15.125:6819/1172219518","cluster_addr":"172.21.15.125:6821/1172219518","heartbeat_back_addr":"172.21.15.125:6825/1172219518","heartbeat_front_addr":"172.21.15.125:6823/1172219518","state":["exists","up"]},{"osd":3,"uuid":"94e49f30-1edd-4b60-8ac7-1b957d1d2dd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":99,"up_from":103,"up_thru":394,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6826","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6827","nonce":2373950101}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6828","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6829","nonce":2373950101}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6832","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6833","nonce":2373950101}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.125:6830","nonce":2373950101},{"type":"v1","addr":"172.21.15.125:6831","nonce":2373950101}]},"public_addr":"172.21.15.125:6827/2373950101","cluster_addr":"172.21.15.125:6829/2373950101","heartbeat_back_addr":"172.21.15.125:6833/2373950101","heartbeat_front_addr":"172.21.15.125:6831/2373950101","state":["exists","up"]},{"osd":4,"uuid":"0bf4cd6e-c2e5-4aa2-805b-4e3265a31dcd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":394,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6800","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6801","nonce":2095368050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6802","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6803","nonce":2095368050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6806","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6807","nonce":2095368050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6804","nonce":2095368050},{"type":"v1","addr":"172.21.15.155:6805","nonce":2095368050}]},"public_addr":"172.21.15.155:6801/2095368050","cluster_addr":"172.21.15.155:6803/2095368050","heartbeat_back_addr":"172.21.15.155:6807/2095368050","heartbeat_front_addr":"172.21.15.155:6805/2095368050","state":["exists","up"]},{"osd":5,"uuid":"79c3a7df-d176-4f67-8de0-1bb46949eeba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":387,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6808","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6809","nonce":395277641}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6810","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6811","nonce":395277641}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6814","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6815","nonce":395277641}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6812","nonce":395277641},{"type":"v1","addr":"172.21.15.155:6813","nonce":395277641}]},"public_addr":"172.21.15.155:6809/395277641","cluster_addr":"172.21.15.155:6811/395277641","heartbeat_back_addr":"172.21.15.155:6815/395277641","heartbeat_front_addr":"172.21.15.155:6813/395277641","state":["exists","up"]},{"osd":6,"uuid":"65cb2125-86cc-4e3c-9bb6-3926ff3db18e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":403,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6816","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6817","nonce":629807062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6818","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6819","nonce":629807062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6822","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6823","nonce":629807062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6820","nonce":629807062},{"type":"v1","addr":"172.21.15.155:6821","nonce":629807062}]},"public_addr":"172.21.15.155:6817/629807062","cluster_addr":"172.21.15.155:6819/629807062","heartbeat_back_addr":"172.21.15.155:6823/629807062","heartbeat_front_addr":"172.21.15.155:6821/629807062","state":["exists","up"]},{"osd":7,"uuid":"cd9d3a50-9683-4779-942e-d41abad90f08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":388,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6824","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6825","nonce":1961936035}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6826","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6827","nonce":1961936035}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6830","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6831","nonce":1961936035}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.155:6828","nonce":1961936035},{"type":"v1","addr":"172.21.15.155:6829","nonce":1961936035}]},"public_addr":"172.21.15.155:6825/1961936035","cluster_addr":"172.21.15.155:6827/1961936035","heartbeat_back_addr":"172.21.15.155:6831/1961936035","heartbeat_front_addr":"172.21.15.155:6829/1961936035","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:45.508384+0000","dead_epoch":84},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:29.584786+0000","dead_epoch":89},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:12.984707+0000","dead_epoch":94},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:58.454701+0000","dead_epoch":99},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:40.057010+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:57:23.142339+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:06.112349+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:58:49.214747+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":4,"to":0}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.4","mappings":[{"from":3,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":3,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.125:0/2757914842":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/1505504676":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/4007395716":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/2669712956":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/4000145196":"2023-10-10T06:08:18.401578+0000","172.21.15.155:6834/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/1272473784":"2023-10-10T05:59:24.160468+0000","172.21.15.125:6800/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/1180505635":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/928092871":"2023-10-10T06:08:18.401578+0000","172.21.15.125:0/457536542":"2023-10-10T06:05:34.904573+0000","172.21.15.125:6800/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.125:6801/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/3175453899":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/26344715":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/4031419935":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/872923687":"2023-10-10T05:59:24.160468+0000","172.21.15.125:0/2544415312":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6800/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/105318502":"2023-10-10T06:08:18.401578+0000","172.21.15.155:0/497767166":"2023-10-10T06:07:14.674105+0000","172.21.15.155:6835/2519956701":"2023-10-10T06:07:14.674105+0000","172.21.15.125:6801/3780184365":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/1760638819":"2023-10-10T05:51:25.812751+0000","172.21.15.125:0/2496192064":"2023-10-10T05:51:55.829150+0000","172.21.15.125:0/4016621680":"2023-10-10T06:05:34.904573+0000","172.21.15.155:0/3959201196":"2023-10-10T06:07:14.674105+0000","172.21.15.125:0/3162814593":"2023-10-10T05:51:55.829150+0000","172.21.15.125:6800/3168262044":"2023-10-10T05:51:25.812751+0000","172.21.15.125:6801/3168262044":"2023-10-10T05:51:25.812751+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:55:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:25 smithi125 bash[77299]: cluster 2023-10-09T06:55:23.606351+0000 mgr.y (mgr.25723) 1532 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:25.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:25 smithi125 bash[77299]: audit 2023-10-09T06:55:25.246863+0000 mon.a (mon.0) 1916 : audit 0 from='client.? 172.21.15.125:0/1896899655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:25.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:25 smithi125 bash[80002]: cluster 2023-10-09T06:55:23.606351+0000 mgr.y (mgr.25723) 1532 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:25.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:25 smithi125 bash[80002]: audit 2023-10-09T06:55:25.246863+0000 mon.a (mon.0) 1916 : audit 0 from='client.? 172.21.15.125:0/1896899655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:25.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:25 smithi155 bash[59495]: cluster 2023-10-09T06:55:23.606351+0000 mgr.y (mgr.25723) 1532 : cluster 0 pgmap v1737: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:25.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:25 smithi155 bash[59495]: audit 2023-10-09T06:55:25.246863+0000 mon.a (mon.0) 1916 : audit 0 from='client.? 172.21.15.125:0/1896899655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:55:26.247 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:55:26.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:55:26.248 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:55:26.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:55:26.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:55:26.249 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:55:26.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:55:26.250 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:55:27.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:27 smithi125 bash[77299]: cluster 2023-10-09T06:55:25.607395+0000 mgr.y (mgr.25723) 1533 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:27.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:27 smithi125 bash[77299]: audit 2023-10-09T06:55:26.976186+0000 mon.a (mon.0) 1917 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:55:27.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:27 smithi125 bash[80002]: cluster 2023-10-09T06:55:25.607395+0000 mgr.y (mgr.25723) 1533 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:27.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:27 smithi125 bash[80002]: audit 2023-10-09T06:55:26.976186+0000 mon.a (mon.0) 1917 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:27 smithi155 bash[59495]: cluster 2023-10-09T06:55:25.607395+0000 mgr.y (mgr.25723) 1533 : cluster 0 pgmap v1738: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:27.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:27 smithi155 bash[59495]: audit 2023-10-09T06:55:26.976186+0000 mon.a (mon.0) 1917 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:55:29.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:29 smithi125 bash[77299]: cluster 2023-10-09T06:55:27.608313+0000 mgr.y (mgr.25723) 1534 : cluster 0 pgmap v1739: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:29.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:29 smithi125 bash[80002]: cluster 2023-10-09T06:55:27.608313+0000 mgr.y (mgr.25723) 1534 : cluster 0 pgmap v1739: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:29.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:29 smithi155 bash[59495]: cluster 2023-10-09T06:55:27.608313+0000 mgr.y (mgr.25723) 1534 : cluster 0 pgmap v1739: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:31.142 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.143 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.143 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.148 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.148 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.149 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.150 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.153 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:31.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:31] "GET /metrics HTTP/1.1" 200 34175 "" "Prometheus/2.33.4" 2023-10-09T06:55:31.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:31 smithi125 bash[80002]: cluster 2023-10-09T06:55:29.609038+0000 mgr.y (mgr.25723) 1535 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:31.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:31 smithi125 bash[77299]: cluster 2023-10-09T06:55:29.609038+0000 mgr.y (mgr.25723) 1535 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:31.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:31 smithi155 bash[59495]: cluster 2023-10-09T06:55:29.609038+0000 mgr.y (mgr.25723) 1535 : cluster 0 pgmap v1740: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:33.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[77299]: cluster 2023-10-09T06:55:31.609820+0000 mgr.y (mgr.25723) 1536 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:33.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[77299]: audit 2023-10-09T06:55:32.314675+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[77299]: audit 2023-10-09T06:55:32.323304+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[77299]: audit 2023-10-09T06:55:33.237937+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[77299]: audit 2023-10-09T06:55:33.244977+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[80002]: cluster 2023-10-09T06:55:31.609820+0000 mgr.y (mgr.25723) 1536 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[80002]: audit 2023-10-09T06:55:32.314675+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[80002]: audit 2023-10-09T06:55:32.323304+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[80002]: audit 2023-10-09T06:55:33.237937+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:33 smithi125 bash[80002]: audit 2023-10-09T06:55:33.244977+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:33 smithi155 bash[59495]: cluster 2023-10-09T06:55:31.609820+0000 mgr.y (mgr.25723) 1536 : cluster 0 pgmap v1741: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:33.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:33 smithi155 bash[59495]: audit 2023-10-09T06:55:32.314675+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:33 smithi155 bash[59495]: audit 2023-10-09T06:55:32.323304+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:33 smithi155 bash[59495]: audit 2023-10-09T06:55:33.237937+0000 mon.a (mon.0) 1920 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:33.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:33 smithi155 bash[59495]: audit 2023-10-09T06:55:33.244977+0000 mon.a (mon.0) 1921 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:34.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[77299]: audit 2023-10-09T06:55:33.622254+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[77299]: audit 2023-10-09T06:55:33.764719+0000 mon.a (mon.0) 1923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[77299]: audit 2023-10-09T06:55:33.766335+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[77299]: audit 2023-10-09T06:55:33.775519+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[80002]: audit 2023-10-09T06:55:33.622254+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[80002]: audit 2023-10-09T06:55:33.764719+0000 mon.a (mon.0) 1923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[80002]: audit 2023-10-09T06:55:33.766335+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:55:34.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:34 smithi125 bash[80002]: audit 2023-10-09T06:55:33.775519+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:34 smithi155 bash[59495]: audit 2023-10-09T06:55:33.622254+0000 mon.a (mon.0) 1922 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:34.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:34 smithi155 bash[59495]: audit 2023-10-09T06:55:33.764719+0000 mon.a (mon.0) 1923 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:55:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:34 smithi155 bash[59495]: audit 2023-10-09T06:55:33.766335+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:55:34.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:34 smithi155 bash[59495]: audit 2023-10-09T06:55:33.775519+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:55:35.414 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:35 smithi125 bash[77299]: cluster 2023-10-09T06:55:33.610587+0000 mgr.y (mgr.25723) 1537 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:35.414 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:35 smithi125 bash[80002]: cluster 2023-10-09T06:55:33.610587+0000 mgr.y (mgr.25723) 1537 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:35.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:35 smithi155 bash[59495]: cluster 2023-10-09T06:55:33.610587+0000 mgr.y (mgr.25723) 1537 : cluster 0 pgmap v1742: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:37.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:37 smithi125 bash[77299]: cluster 2023-10-09T06:55:35.611154+0000 mgr.y (mgr.25723) 1538 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:37.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:37 smithi125 bash[80002]: cluster 2023-10-09T06:55:35.611154+0000 mgr.y (mgr.25723) 1538 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:37.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:37 smithi155 bash[59495]: cluster 2023-10-09T06:55:35.611154+0000 mgr.y (mgr.25723) 1538 : cluster 0 pgmap v1743: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:39.609 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:39 smithi125 bash[77299]: cluster 2023-10-09T06:55:37.611750+0000 mgr.y (mgr.25723) 1539 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:39.610 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:39 smithi125 bash[80002]: cluster 2023-10-09T06:55:37.611750+0000 mgr.y (mgr.25723) 1539 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:39.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:39 smithi155 bash[59495]: cluster 2023-10-09T06:55:37.611750+0000 mgr.y (mgr.25723) 1539 : cluster 0 pgmap v1744: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:41.557 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:41 smithi125 bash[77299]: cluster 2023-10-09T06:55:39.612289+0000 mgr.y (mgr.25723) 1540 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:41.557 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:41 smithi125 bash[80002]: cluster 2023-10-09T06:55:39.612289+0000 mgr.y (mgr.25723) 1540 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:41.557 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:41] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.33.4" 2023-10-09T06:55:41.644 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:41 smithi155 bash[59495]: cluster 2023-10-09T06:55:39.612289+0000 mgr.y (mgr.25723) 1540 : cluster 0 pgmap v1745: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:43.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:43 smithi125 bash[80002]: cluster 2023-10-09T06:55:41.613124+0000 mgr.y (mgr.25723) 1541 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:43.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:43 smithi125 bash[77299]: cluster 2023-10-09T06:55:41.613124+0000 mgr.y (mgr.25723) 1541 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:43.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:43 smithi155 bash[59495]: cluster 2023-10-09T06:55:41.613124+0000 mgr.y (mgr.25723) 1541 : cluster 0 pgmap v1746: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:45.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:45 smithi125 bash[77299]: cluster 2023-10-09T06:55:43.613904+0000 mgr.y (mgr.25723) 1542 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:45.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:45 smithi125 bash[80002]: cluster 2023-10-09T06:55:43.613904+0000 mgr.y (mgr.25723) 1542 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:45.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:45 smithi155 bash[59495]: cluster 2023-10-09T06:55:43.613904+0000 mgr.y (mgr.25723) 1542 : cluster 0 pgmap v1747: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:47.586 INFO:teuthology.orchestra.run.smithi125.stdout:128849019592 2023-10-09T06:55:47.589 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:55:47.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:47 smithi125 bash[77299]: cluster 2023-10-09T06:55:45.614362+0000 mgr.y (mgr.25723) 1543 : cluster 0 pgmap v1748: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:47.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:47 smithi125 bash[80002]: cluster 2023-10-09T06:55:45.614362+0000 mgr.y (mgr.25723) 1543 : cluster 0 pgmap v1748: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:47.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:47 smithi155 bash[59495]: cluster 2023-10-09T06:55:45.614362+0000 mgr.y (mgr.25723) 1543 : cluster 0 pgmap v1748: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:49.638 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:49 smithi125 bash[77299]: cluster 2023-10-09T06:55:47.614931+0000 mgr.y (mgr.25723) 1544 : cluster 0 pgmap v1749: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:49.639 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:49 smithi125 bash[77299]: audit 2023-10-09T06:55:48.622484+0000 mon.a (mon.0) 1926 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:49.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:49 smithi125 bash[80002]: cluster 2023-10-09T06:55:47.614931+0000 mgr.y (mgr.25723) 1544 : cluster 0 pgmap v1749: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:49.639 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:49 smithi125 bash[80002]: audit 2023-10-09T06:55:48.622484+0000 mon.a (mon.0) 1926 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:49 smithi155 bash[59495]: cluster 2023-10-09T06:55:47.614931+0000 mgr.y (mgr.25723) 1544 : cluster 0 pgmap v1749: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:49.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:49 smithi155 bash[59495]: audit 2023-10-09T06:55:48.622484+0000 mon.a (mon.0) 1926 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:55:50.000 INFO:teuthology.orchestra.run.smithi125.stdout:399431959035 2023-10-09T06:55:50.001 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:55:50.392 INFO:teuthology.orchestra.run.smithi125.stdout:206158430894 2023-10-09T06:55:50.392 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:55:51.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:51 smithi125 bash[77299]: cluster 2023-10-09T06:55:49.615769+0000 mgr.y (mgr.25723) 1545 : cluster 0 pgmap v1750: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:51.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:55:51 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:55:51] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.33.4" 2023-10-09T06:55:51.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:51 smithi125 bash[80002]: cluster 2023-10-09T06:55:49.615769+0000 mgr.y (mgr.25723) 1545 : cluster 0 pgmap v1750: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:51.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:51 smithi155 bash[59495]: cluster 2023-10-09T06:55:49.615769+0000 mgr.y (mgr.25723) 1545 : cluster 0 pgmap v1750: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:52.306 INFO:teuthology.orchestra.run.smithi125.stdout:154618823359 2023-10-09T06:55:52.307 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:55:53.042 INFO:teuthology.orchestra.run.smithi125.stdout:180388627126 2023-10-09T06:55:53.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:55:53.301 INFO:teuthology.orchestra.run.smithi125.stdout:377957122562 2023-10-09T06:55:53.302 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:55:53.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:53 smithi125 bash[77299]: cluster 2023-10-09T06:55:51.616420+0000 mgr.y (mgr.25723) 1546 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:53.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:53 smithi125 bash[80002]: cluster 2023-10-09T06:55:51.616420+0000 mgr.y (mgr.25723) 1546 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:53.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:53 smithi155 bash[59495]: cluster 2023-10-09T06:55:51.616420+0000 mgr.y (mgr.25723) 1546 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:53.917 INFO:teuthology.orchestra.run.smithi125.stdout:442381631984 2023-10-09T06:55:53.917 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:55:53.928 INFO:teuthology.orchestra.run.smithi125.stdout:420906795510 2023-10-09T06:55:53.929 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:55:55.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:55 smithi125 bash[77299]: cluster 2023-10-09T06:55:53.617138+0000 mgr.y (mgr.25723) 1547 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:55.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:55 smithi125 bash[80002]: cluster 2023-10-09T06:55:53.617138+0000 mgr.y (mgr.25723) 1547 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:55.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:55 smithi155 bash[59495]: cluster 2023-10-09T06:55:53.617138+0000 mgr.y (mgr.25723) 1547 : cluster 0 pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:56.404 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:56.404 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:56.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:56.405 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:57.417 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:57.417 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:57.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:57 smithi125 bash[77299]: cluster 2023-10-09T06:55:55.618162+0000 mgr.y (mgr.25723) 1548 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:57.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:57 smithi125 bash[80002]: cluster 2023-10-09T06:55:55.618162+0000 mgr.y (mgr.25723) 1548 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:57.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:57 smithi155 bash[59495]: cluster 2023-10-09T06:55:55.618162+0000 mgr.y (mgr.25723) 1548 : cluster 0 pgmap v1753: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:58.427 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:58.428 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:55:59.637 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:55:59 smithi125 bash[77299]: cluster 2023-10-09T06:55:57.618894+0000 mgr.y (mgr.25723) 1549 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:59.638 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:55:59 smithi125 bash[80002]: cluster 2023-10-09T06:55:57.618894+0000 mgr.y (mgr.25723) 1549 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:55:59.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:55:59 smithi155 bash[59495]: cluster 2023-10-09T06:55:57.618894+0000 mgr.y (mgr.25723) 1549 : cluster 0 pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:01.642 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:01 smithi125 bash[80002]: cluster 2023-10-09T06:55:59.619501+0000 mgr.y (mgr.25723) 1550 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:01.643 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:01 smithi125 bash[77299]: cluster 2023-10-09T06:55:59.619501+0000 mgr.y (mgr.25723) 1550 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:01.643 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:01 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:56:01] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.33.4" 2023-10-09T06:56:01.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:01 smithi155 bash[59495]: cluster 2023-10-09T06:55:59.619501+0000 mgr.y (mgr.25723) 1550 : cluster 0 pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:03.643 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:03 smithi155 bash[59495]: cluster 2023-10-09T06:56:01.620008+0000 mgr.y (mgr.25723) 1551 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:03.884 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:03 smithi125 bash[77299]: cluster 2023-10-09T06:56:01.620008+0000 mgr.y (mgr.25723) 1551 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:03.884 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:03 smithi125 bash[80002]: cluster 2023-10-09T06:56:01.620008+0000 mgr.y (mgr.25723) 1551 : cluster 0 pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:04.582 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:04 smithi125 bash[77299]: audit 2023-10-09T06:56:03.623368+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:04.585 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:04 smithi125 bash[80002]: audit 2023-10-09T06:56:03.623368+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:04.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:04 smithi155 bash[59495]: audit 2023-10-09T06:56:03.623368+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:05.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:05 smithi125 bash[80002]: cluster 2023-10-09T06:56:03.620430+0000 mgr.y (mgr.25723) 1552 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:05.887 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:05 smithi125 bash[77299]: cluster 2023-10-09T06:56:03.620430+0000 mgr.y (mgr.25723) 1552 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:05.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:05 smithi155 bash[59495]: cluster 2023-10-09T06:56:03.620430+0000 mgr.y (mgr.25723) 1552 : cluster 0 pgmap v1757: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:07.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:07 smithi125 bash[77299]: cluster 2023-10-09T06:56:05.621155+0000 mgr.y (mgr.25723) 1553 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:07.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:07 smithi125 bash[80002]: cluster 2023-10-09T06:56:05.621155+0000 mgr.y (mgr.25723) 1553 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:07.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:07 smithi155 bash[59495]: cluster 2023-10-09T06:56:05.621155+0000 mgr.y (mgr.25723) 1553 : cluster 0 pgmap v1758: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:09.676 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:09 smithi125 bash[77299]: cluster 2023-10-09T06:56:07.621712+0000 mgr.y (mgr.25723) 1554 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:09.676 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:09 smithi125 bash[80002]: cluster 2023-10-09T06:56:07.621712+0000 mgr.y (mgr.25723) 1554 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:09.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:09 smithi155 bash[59495]: cluster 2023-10-09T06:56:07.621712+0000 mgr.y (mgr.25723) 1554 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:11.029 INFO:teuthology.orchestra.run.smithi125.stdout:128849019597 2023-10-09T06:56:11.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:11 smithi125 bash[77299]: cluster 2023-10-09T06:56:09.622551+0000 mgr.y (mgr.25723) 1555 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:11.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:11 smithi125 bash[77299]: audit 2023-10-09T06:56:11.026743+0000 mon.a (mon.0) 1928 : audit 0 from='client.? 172.21.15.125:0/1055272793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:56:11.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:11 smithi125 bash[80002]: cluster 2023-10-09T06:56:09.622551+0000 mgr.y (mgr.25723) 1555 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:11.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:11 smithi125 bash[80002]: audit 2023-10-09T06:56:11.026743+0000 mon.a (mon.0) 1928 : audit 0 from='client.? 172.21.15.125:0/1055272793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:56:11.637 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:11 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:56:11] "GET /metrics HTTP/1.1" 200 34176 "" "Prometheus/2.33.4" 2023-10-09T06:56:11.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:11 smithi155 bash[59495]: cluster 2023-10-09T06:56:09.622551+0000 mgr.y (mgr.25723) 1555 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:11.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:11 smithi155 bash[59495]: audit 2023-10-09T06:56:11.026743+0000 mon.a (mon.0) 1928 : audit 0 from='client.? 172.21.15.125:0/1055272793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:56:13.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:13 smithi125 bash[77299]: cluster 2023-10-09T06:56:11.623286+0000 mgr.y (mgr.25723) 1556 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:13.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:13 smithi125 bash[80002]: cluster 2023-10-09T06:56:11.623286+0000 mgr.y (mgr.25723) 1556 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:13.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:13 smithi155 bash[59495]: cluster 2023-10-09T06:56:11.623286+0000 mgr.y (mgr.25723) 1556 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:14.598 INFO:teuthology.orchestra.run.smithi125.stdout:206158430899 2023-10-09T06:56:15.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:15 smithi125 bash[77299]: cluster 2023-10-09T06:56:13.623888+0000 mgr.y (mgr.25723) 1557 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:15.699 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:15 smithi125 bash[77299]: audit 2023-10-09T06:56:14.596364+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.125:0/897345464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:56:15.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:15 smithi125 bash[80002]: cluster 2023-10-09T06:56:13.623888+0000 mgr.y (mgr.25723) 1557 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:15.700 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:15 smithi125 bash[80002]: audit 2023-10-09T06:56:14.596364+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.125:0/897345464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:56:15.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:15 smithi155 bash[59495]: cluster 2023-10-09T06:56:13.623888+0000 mgr.y (mgr.25723) 1557 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:15.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:15 smithi155 bash[59495]: audit 2023-10-09T06:56:14.596364+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.125:0/897345464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:56:16.092 INFO:teuthology.orchestra.run.smithi125.stdout:154618823364 2023-10-09T06:56:16.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:16 smithi125 bash[77299]: audit 2023-10-09T06:56:16.089566+0000 mon.a (mon.0) 1929 : audit 0 from='client.? 172.21.15.125:0/1815260363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:56:16.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:16 smithi125 bash[80002]: audit 2023-10-09T06:56:16.089566+0000 mon.a (mon.0) 1929 : audit 0 from='client.? 172.21.15.125:0/1815260363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:56:16.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:16 smithi155 bash[59495]: audit 2023-10-09T06:56:16.089566+0000 mon.a (mon.0) 1929 : audit 0 from='client.? 172.21.15.125:0/1815260363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:56:17.250 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019592 got 128849019597 for osd.4 2023-10-09T06:56:17.250 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:17.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:17 smithi125 bash[77299]: cluster 2023-10-09T06:56:15.624508+0000 mgr.y (mgr.25723) 1558 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:17.637 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:17 smithi125 bash[80002]: cluster 2023-10-09T06:56:15.624508+0000 mgr.y (mgr.25723) 1558 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:17.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:17 smithi155 bash[59495]: cluster 2023-10-09T06:56:15.624508+0000 mgr.y (mgr.25723) 1558 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:18.611 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430894 got 206158430899 for osd.7 2023-10-09T06:56:18.611 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:19.177 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823359 got 154618823364 for osd.5 2023-10-09T06:56:19.178 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:19.474 INFO:teuthology.orchestra.run.smithi125.stdout:442381631990 2023-10-09T06:56:19.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:19 smithi125 bash[77299]: cluster 2023-10-09T06:56:17.625328+0000 mgr.y (mgr.25723) 1559 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:19.657 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:19 smithi125 bash[77299]: audit 2023-10-09T06:56:18.623461+0000 mon.a (mon.0) 1930 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:19.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:19 smithi125 bash[80002]: cluster 2023-10-09T06:56:17.625328+0000 mgr.y (mgr.25723) 1559 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:19.658 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:19 smithi125 bash[80002]: audit 2023-10-09T06:56:18.623461+0000 mon.a (mon.0) 1930 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:19.784 INFO:teuthology.orchestra.run.smithi125.stdout:399431959042 2023-10-09T06:56:19.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:19 smithi155 bash[59495]: cluster 2023-10-09T06:56:17.625328+0000 mgr.y (mgr.25723) 1559 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:19.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:19 smithi155 bash[59495]: audit 2023-10-09T06:56:18.623461+0000 mon.a (mon.0) 1930 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:20.040 INFO:teuthology.orchestra.run.smithi125.stdout:180388627132 2023-10-09T06:56:20.622 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[80002]: audit 2023-10-09T06:56:19.471942+0000 mon.a (mon.0) 1931 : audit 0 from='client.? 172.21.15.125:0/984972610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:56:20.623 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[80002]: audit 2023-10-09T06:56:19.781562+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.125:0/280620989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:56:20.623 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[80002]: audit 2023-10-09T06:56:20.034654+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.125:0/1526063045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:56:20.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[77299]: audit 2023-10-09T06:56:19.471942+0000 mon.a (mon.0) 1931 : audit 0 from='client.? 172.21.15.125:0/984972610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:56:20.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[77299]: audit 2023-10-09T06:56:19.781562+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.125:0/280620989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:56:20.623 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:20 smithi125 bash[77299]: audit 2023-10-09T06:56:20.034654+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.125:0/1526063045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:56:20.714 INFO:teuthology.orchestra.run.smithi125.stdout:420906795516 2023-10-09T06:56:20.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:20 smithi155 bash[59495]: audit 2023-10-09T06:56:19.471942+0000 mon.a (mon.0) 1931 : audit 0 from='client.? 172.21.15.125:0/984972610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:56:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:20 smithi155 bash[59495]: audit 2023-10-09T06:56:19.781562+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.125:0/280620989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:56:20.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:20 smithi155 bash[59495]: audit 2023-10-09T06:56:20.034654+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.125:0/1526063045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:56:21.084 INFO:teuthology.orchestra.run.smithi125.stdout:377957122568 2023-10-09T06:56:21.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[77299]: cluster 2023-10-09T06:56:19.626218+0000 mgr.y (mgr.25723) 1560 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:21.524 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[77299]: audit 2023-10-09T06:56:20.711529+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.125:0/700167196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:56:21.524 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:56:21] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:56:21.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[77299]: audit 2023-10-09T06:56:21.082464+0000 mon.a (mon.0) 1932 : audit 0 from='client.? 172.21.15.125:0/2845128396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:56:21.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[80002]: cluster 2023-10-09T06:56:19.626218+0000 mgr.y (mgr.25723) 1560 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:21.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[80002]: audit 2023-10-09T06:56:20.711529+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.125:0/700167196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:56:21.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:21 smithi125 bash[80002]: audit 2023-10-09T06:56:21.082464+0000 mon.a (mon.0) 1932 : audit 0 from='client.? 172.21.15.125:0/2845128396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:56:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:21 smithi155 bash[59495]: cluster 2023-10-09T06:56:19.626218+0000 mgr.y (mgr.25723) 1560 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:21.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:21 smithi155 bash[59495]: audit 2023-10-09T06:56:20.711529+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.125:0/700167196' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:56:21.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:21 smithi155 bash[59495]: audit 2023-10-09T06:56:21.082464+0000 mon.a (mon.0) 1932 : audit 0 from='client.? 172.21.15.125:0/2845128396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:56:23.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631984 got 442381631990 for osd.3 2023-10-09T06:56:23.097 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:23.315 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627126 got 180388627132 for osd.6 2023-10-09T06:56:23.316 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:23.316 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431959035 got 399431959042 for osd.1 2023-10-09T06:56:23.316 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:23.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:23 smithi125 bash[77299]: cluster 2023-10-09T06:56:21.626946+0000 mgr.y (mgr.25723) 1561 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:23.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:23 smithi125 bash[80002]: cluster 2023-10-09T06:56:21.626946+0000 mgr.y (mgr.25723) 1561 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:23.895 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:23 smithi155 bash[59495]: cluster 2023-10-09T06:56:21.626946+0000 mgr.y (mgr.25723) 1561 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:24.116 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795510 got 420906795516 for osd.2 2023-10-09T06:56:24.116 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:24.212 INFO:tasks.cephadm.ceph_manager.ceph:need seq 377957122562 got 377957122568 for osd.0 2023-10-09T06:56:24.212 DEBUG:teuthology.parallel:result is None 2023-10-09T06:56:24.213 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:56:24.213 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:56:25.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:25 smithi125 bash[77299]: cluster 2023-10-09T06:56:23.627663+0000 mgr.y (mgr.25723) 1562 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:25.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:25 smithi125 bash[80002]: cluster 2023-10-09T06:56:23.627663+0000 mgr.y (mgr.25723) 1562 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:25.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:25 smithi155 bash[59495]: cluster 2023-10-09T06:56:23.627663+0000 mgr.y (mgr.25723) 1562 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:27.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:27 smithi125 bash[77299]: cluster 2023-10-09T06:56:25.628582+0000 mgr.y (mgr.25723) 1563 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:27.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:27 smithi125 bash[80002]: cluster 2023-10-09T06:56:25.628582+0000 mgr.y (mgr.25723) 1563 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:27.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:27 smithi155 bash[59495]: cluster 2023-10-09T06:56:25.628582+0000 mgr.y (mgr.25723) 1563 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:28.988 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:56:29.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:29 smithi125 bash[77299]: cluster 2023-10-09T06:56:27.629323+0000 mgr.y (mgr.25723) 1564 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:29.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:29 smithi125 bash[80002]: cluster 2023-10-09T06:56:27.629323+0000 mgr.y (mgr.25723) 1564 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:29.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:29 smithi155 bash[59495]: cluster 2023-10-09T06:56:27.629323+0000 mgr.y (mgr.25723) 1564 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:31.470 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:31 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:56:31] "GET /metrics HTTP/1.1" 200 34177 "" "Prometheus/2.33.4" 2023-10-09T06:56:31.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:31 smithi125 bash[77299]: cluster 2023-10-09T06:56:29.630212+0000 mgr.y (mgr.25723) 1565 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:31.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:31 smithi125 bash[80002]: cluster 2023-10-09T06:56:29.630212+0000 mgr.y (mgr.25723) 1565 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:31.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:31 smithi155 bash[59495]: cluster 2023-10-09T06:56:29.630212+0000 mgr.y (mgr.25723) 1565 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:33.065 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:56:33.067 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:56:33.636 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:33 smithi125 bash[77299]: cluster 2023-10-09T06:56:31.631118+0000 mgr.y (mgr.25723) 1566 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:33.636 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:33 smithi125 bash[80002]: cluster 2023-10-09T06:56:31.631118+0000 mgr.y (mgr.25723) 1566 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:33.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:33 smithi155 bash[59495]: cluster 2023-10-09T06:56:31.631118+0000 mgr.y (mgr.25723) 1566 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:34.015 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1771,"stamp":"2023-10-09T06:56:31.630539+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12192,"num_read_kb":25881,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":595,"num_bytes_recovered":20351220,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6367264,"kb_used_data":57632,"kb_used_omap":30,"kb_used_meta":6309537,"kb_avail":743593952,"statfs":{"total":767960285184,"available":761440206848,"internally_reserved":0,"allocated":59015168,"data_stored":50650473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":6460965999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004896"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":423,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455528+0000","last_change":"2023-10-09T06:52:41.261259+0000","last_active":"2023-10-09T06:54:47.455528+0000","last_peered":"2023-10-09T06:54:47.455528+0000","last_clean":"2023-10-09T06:54:47.455528+0000","last_became_active":"2023-10-09T06:52:41.260731+0000","last_became_peered":"2023-10-09T06:52:41.260731+0000","last_unstale":"2023-10-09T06:54:47.455528+0000","last_undegraded":"2023-10-09T06:54:47.455528+0000","last_fullsized":"2023-10-09T06:54:47.455528+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670457+0000","last_change":"2023-10-09T06:52:41.258305+0000","last_active":"2023-10-09T06:54:56.670457+0000","last_peered":"2023-10-09T06:54:56.670457+0000","last_clean":"2023-10-09T06:54:56.670457+0000","last_became_active":"2023-10-09T06:52:41.257911+0000","last_became_peered":"2023-10-09T06:52:41.257911+0000","last_unstale":"2023-10-09T06:54:56.670457+0000","last_undegraded":"2023-10-09T06:54:56.670457+0000","last_fullsized":"2023-10-09T06:54:56.670457+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:24.252303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":70,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.798666+0000","last_change":"2023-10-09T06:52:53.666252+0000","last_active":"2023-10-09T06:54:57.798666+0000","last_peered":"2023-10-09T06:54:57.798666+0000","last_clean":"2023-10-09T06:54:57.798666+0000","last_became_active":"2023-10-09T06:52:53.665868+0000","last_became_peered":"2023-10-09T06:52:53.665868+0000","last_unstale":"2023-10-09T06:54:57.798666+0000","last_undegraded":"2023-10-09T06:54:57.798666+0000","last_fullsized":"2023-10-09T06:54:57.798666+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:50.509786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450700+0000","last_change":"2023-10-09T06:52:29.653622+0000","last_active":"2023-10-09T06:54:47.450700+0000","last_peered":"2023-10-09T06:54:47.450700+0000","last_clean":"2023-10-09T06:54:47.450700+0000","last_became_active":"2023-10-09T06:52:29.652927+0000","last_became_peered":"2023-10-09T06:52:29.652927+0000","last_unstale":"2023-10-09T06:54:47.450700+0000","last_undegraded":"2023-10-09T06:54:47.450700+0000","last_fullsized":"2023-10-09T06:54:47.450700+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:01.072762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669510+0000","last_change":"2023-10-09T06:52:41.249371+0000","last_active":"2023-10-09T06:54:56.669510+0000","last_peered":"2023-10-09T06:54:56.669510+0000","last_clean":"2023-10-09T06:54:56.669510+0000","last_became_active":"2023-10-09T06:52:41.249008+0000","last_became_peered":"2023-10-09T06:52:41.249008+0000","last_unstale":"2023-10-09T06:54:56.669510+0000","last_undegraded":"2023-10-09T06:54:56.669510+0000","last_fullsized":"2023-10-09T06:54:56.669510+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:57.074206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666433+0000","last_change":"2023-10-09T06:51:41.234140+0000","last_active":"2023-10-09T06:54:46.666433+0000","last_peered":"2023-10-09T06:54:46.666433+0000","last_clean":"2023-10-09T06:54:46.666433+0000","last_became_active":"2023-10-09T06:51:41.233841+0000","last_became_peered":"2023-10-09T06:51:41.233841+0000","last_unstale":"2023-10-09T06:54:46.666433+0000","last_undegraded":"2023-10-09T06:54:46.666433+0000","last_fullsized":"2023-10-09T06:54:46.666433+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":284,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123972+0000","last_change":"2023-10-09T06:52:29.658396+0000","last_active":"2023-10-09T06:54:57.123972+0000","last_peered":"2023-10-09T06:54:57.123972+0000","last_clean":"2023-10-09T06:54:57.123972+0000","last_became_active":"2023-10-09T06:52:29.658046+0000","last_became_peered":"2023-10-09T06:52:29.658046+0000","last_unstale":"2023-10-09T06:54:57.123972+0000","last_undegraded":"2023-10-09T06:54:57.123972+0000","last_fullsized":"2023-10-09T06:54:57.123972+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:42.469186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669512+0000","last_change":"2023-10-09T06:52:41.255390+0000","last_active":"2023-10-09T06:54:56.669512+0000","last_peered":"2023-10-09T06:54:56.669512+0000","last_clean":"2023-10-09T06:54:56.669512+0000","last_became_active":"2023-10-09T06:52:41.254984+0000","last_became_peered":"2023-10-09T06:52:41.254984+0000","last_unstale":"2023-10-09T06:54:56.669512+0000","last_undegraded":"2023-10-09T06:54:56.669512+0000","last_fullsized":"2023-10-09T06:54:56.669512+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:14.258020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":57,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455161+0000","last_change":"2023-10-09T06:52:41.261238+0000","last_active":"2023-10-09T06:54:47.455161+0000","last_peered":"2023-10-09T06:54:47.455161+0000","last_clean":"2023-10-09T06:54:47.455161+0000","last_became_active":"2023-10-09T06:52:41.260852+0000","last_became_peered":"2023-10-09T06:52:41.260852+0000","last_unstale":"2023-10-09T06:54:47.455161+0000","last_undegraded":"2023-10-09T06:54:47.455161+0000","last_fullsized":"2023-10-09T06:54:47.455161+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669737+0000","last_change":"2023-10-09T06:51:08.220112+0000","last_active":"2023-10-09T06:54:56.669737+0000","last_peered":"2023-10-09T06:54:56.669737+0000","last_clean":"2023-10-09T06:54:56.669737+0000","last_became_active":"2023-10-09T06:51:08.219059+0000","last_became_peered":"2023-10-09T06:51:08.219059+0000","last_unstale":"2023-10-09T06:54:56.669737+0000","last_undegraded":"2023-10-09T06:54:56.669737+0000","last_fullsized":"2023-10-09T06:54:56.669737+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:12.671358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":333,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181209+0000","last_change":"2023-10-09T06:51:08.360804+0000","last_active":"2023-10-09T06:54:47.181209+0000","last_peered":"2023-10-09T06:54:47.181209+0000","last_clean":"2023-10-09T06:54:47.181209+0000","last_became_active":"2023-10-09T06:51:08.221588+0000","last_became_peered":"2023-10-09T06:51:08.221588+0000","last_unstale":"2023-10-09T06:54:47.181209+0000","last_undegraded":"2023-10-09T06:54:47.181209+0000","last_fullsized":"2023-10-09T06:54:47.181209+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":186,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799561+0000","last_change":"2023-10-09T06:52:54.127043+0000","last_active":"2023-10-09T06:54:57.799561+0000","last_peered":"2023-10-09T06:54:57.799561+0000","last_clean":"2023-10-09T06:54:57.799561+0000","last_became_active":"2023-10-09T06:52:54.126228+0000","last_became_peered":"2023-10-09T06:52:54.126228+0000","last_unstale":"2023-10-09T06:54:57.799561+0000","last_undegraded":"2023-10-09T06:54:57.799561+0000","last_fullsized":"2023-10-09T06:54:57.799561+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:30.427085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":492,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455880+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:54:47.455880+0000","last_peered":"2023-10-09T06:54:47.455880+0000","last_clean":"2023-10-09T06:54:47.455880+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:54:47.455880+0000","last_undegraded":"2023-10-09T06:54:47.455880+0000","last_fullsized":"2023-10-09T06:54:47.455880+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":451,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455837+0000","last_change":"2023-10-09T06:53:22.252174+0000","last_active":"2023-10-09T06:54:47.455837+0000","last_peered":"2023-10-09T06:54:47.455837+0000","last_clean":"2023-10-09T06:54:47.455837+0000","last_became_active":"2023-10-09T06:53:22.251093+0000","last_became_peered":"2023-10-09T06:53:22.251093+0000","last_unstale":"2023-10-09T06:54:47.455837+0000","last_undegraded":"2023-10-09T06:54:47.455837+0000","last_fullsized":"2023-10-09T06:54:47.455837+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":333,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180598+0000","last_change":"2023-10-09T06:51:53.829980+0000","last_active":"2023-10-09T06:54:47.180598+0000","last_peered":"2023-10-09T06:54:47.180598+0000","last_clean":"2023-10-09T06:54:47.180598+0000","last_became_active":"2023-10-09T06:51:53.655619+0000","last_became_peered":"2023-10-09T06:51:53.655619+0000","last_unstale":"2023-10-09T06:54:47.180598+0000","last_undegraded":"2023-10-09T06:54:47.180598+0000","last_fullsized":"2023-10-09T06:54:47.180598+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":44,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455204+0000","last_change":"2023-10-09T06:52:41.266974+0000","last_active":"2023-10-09T06:54:47.455204+0000","last_peered":"2023-10-09T06:54:47.455204+0000","last_clean":"2023-10-09T06:54:47.455204+0000","last_became_active":"2023-10-09T06:52:41.266599+0000","last_became_peered":"2023-10-09T06:52:41.266599+0000","last_unstale":"2023-10-09T06:54:47.455204+0000","last_undegraded":"2023-10-09T06:54:47.455204+0000","last_fullsized":"2023-10-09T06:54:47.455204+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":248,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:53.059335+0000","last_change":"2023-10-09T06:52:29.651724+0000","last_active":"2023-10-09T06:54:53.059335+0000","last_peered":"2023-10-09T06:54:53.059335+0000","last_clean":"2023-10-09T06:54:53.059335+0000","last_became_active":"2023-10-09T06:52:29.651303+0000","last_became_peered":"2023-10-09T06:52:29.651303+0000","last_unstale":"2023-10-09T06:54:53.059335+0000","last_undegraded":"2023-10-09T06:54:53.059335+0000","last_fullsized":"2023-10-09T06:54:53.059335+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:53.112045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455477+0000","last_change":"2023-10-09T06:51:41.236068+0000","last_active":"2023-10-09T06:54:47.455477+0000","last_peered":"2023-10-09T06:54:47.455477+0000","last_clean":"2023-10-09T06:54:47.455477+0000","last_became_active":"2023-10-09T06:51:41.235740+0000","last_became_peered":"2023-10-09T06:51:41.235740+0000","last_unstale":"2023-10-09T06:54:47.455477+0000","last_undegraded":"2023-10-09T06:54:47.455477+0000","last_fullsized":"2023-10-09T06:54:47.455477+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":351,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180589+0000","last_change":"2023-10-09T06:45:05.141190+0000","last_active":"2023-10-09T06:54:47.180589+0000","last_peered":"2023-10-09T06:54:47.180589+0000","last_clean":"2023-10-09T06:54:47.180589+0000","last_became_active":"2023-10-09T06:45:05.140792+0000","last_became_peered":"2023-10-09T06:45:05.140792+0000","last_unstale":"2023-10-09T06:54:47.180589+0000","last_undegraded":"2023-10-09T06:54:47.180589+0000","last_fullsized":"2023-10-09T06:54:47.180589+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":248,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123929+0000","last_change":"2023-10-09T06:52:29.653842+0000","last_active":"2023-10-09T06:54:57.123929+0000","last_peered":"2023-10-09T06:54:57.123929+0000","last_clean":"2023-10-09T06:54:57.123929+0000","last_became_active":"2023-10-09T06:52:29.652933+0000","last_became_peered":"2023-10-09T06:52:29.652933+0000","last_unstale":"2023-10-09T06:54:57.123929+0000","last_undegraded":"2023-10-09T06:54:57.123929+0000","last_fullsized":"2023-10-09T06:54:57.123929+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:26.998073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":239,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666109+0000","last_change":"2023-10-09T06:51:41.413465+0000","last_active":"2023-10-09T06:54:46.666109+0000","last_peered":"2023-10-09T06:54:46.666109+0000","last_clean":"2023-10-09T06:54:46.666109+0000","last_became_active":"2023-10-09T06:51:41.238496+0000","last_became_peered":"2023-10-09T06:51:41.238496+0000","last_unstale":"2023-10-09T06:54:46.666109+0000","last_undegraded":"2023-10-09T06:54:46.666109+0000","last_fullsized":"2023-10-09T06:54:46.666109+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670753+0000","last_change":"2023-10-09T06:53:22.251307+0000","last_active":"2023-10-09T06:54:56.670753+0000","last_peered":"2023-10-09T06:54:56.670753+0000","last_clean":"2023-10-09T06:54:56.670753+0000","last_became_active":"2023-10-09T06:53:22.250530+0000","last_became_peered":"2023-10-09T06:53:22.250530+0000","last_unstale":"2023-10-09T06:54:56.670753+0000","last_undegraded":"2023-10-09T06:54:56.670753+0000","last_fullsized":"2023-10-09T06:54:56.670753+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:32.096800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":301,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181329+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:54:47.181329+0000","last_peered":"2023-10-09T06:54:47.181329+0000","last_clean":"2023-10-09T06:54:47.181329+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:54:47.181329+0000","last_undegraded":"2023-10-09T06:54:47.181329+0000","last_fullsized":"2023-10-09T06:54:47.181329+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.454943+0000","last_change":"2023-10-09T06:51:41.231467+0000","last_active":"2023-10-09T06:54:47.454943+0000","last_peered":"2023-10-09T06:54:47.454943+0000","last_clean":"2023-10-09T06:54:47.454943+0000","last_became_active":"2023-10-09T06:51:41.231125+0000","last_became_peered":"2023-10-09T06:51:41.231125+0000","last_unstale":"2023-10-09T06:54:47.454943+0000","last_undegraded":"2023-10-09T06:54:47.454943+0000","last_fullsized":"2023-10-09T06:54:47.454943+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":353,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181152+0000","last_change":"2023-10-09T06:51:26.276815+0000","last_active":"2023-10-09T06:54:47.181152+0000","last_peered":"2023-10-09T06:54:47.181152+0000","last_clean":"2023-10-09T06:54:47.181152+0000","last_became_active":"2023-10-09T06:51:26.229980+0000","last_became_peered":"2023-10-09T06:51:26.229980+0000","last_unstale":"2023-10-09T06:54:47.181152+0000","last_undegraded":"2023-10-09T06:54:47.181152+0000","last_fullsized":"2023-10-09T06:54:47.181152+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":430,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455358+0000","last_change":"2023-10-09T06:52:41.261783+0000","last_active":"2023-10-09T06:54:47.455358+0000","last_peered":"2023-10-09T06:54:47.455358+0000","last_clean":"2023-10-09T06:54:47.455358+0000","last_became_active":"2023-10-09T06:52:41.261385+0000","last_became_peered":"2023-10-09T06:52:41.261385+0000","last_unstale":"2023-10-09T06:54:47.455358+0000","last_undegraded":"2023-10-09T06:54:47.455358+0000","last_fullsized":"2023-10-09T06:54:47.455358+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":405,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180971+0000","last_change":"2023-10-09T06:51:41.458272+0000","last_active":"2023-10-09T06:54:47.180971+0000","last_peered":"2023-10-09T06:54:47.180971+0000","last_clean":"2023-10-09T06:54:47.180971+0000","last_became_active":"2023-10-09T06:51:41.236211+0000","last_became_peered":"2023-10-09T06:51:41.236211+0000","last_unstale":"2023-10-09T06:54:47.180971+0000","last_undegraded":"2023-10-09T06:54:47.180971+0000","last_fullsized":"2023-10-09T06:54:47.180971+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666040+0000","last_change":"2023-10-09T06:52:21.262418+0000","last_active":"2023-10-09T06:54:46.666040+0000","last_peered":"2023-10-09T06:54:46.666040+0000","last_clean":"2023-10-09T06:54:46.666040+0000","last_became_active":"2023-10-09T06:52:21.261834+0000","last_became_peered":"2023-10-09T06:52:21.261834+0000","last_unstale":"2023-10-09T06:54:46.666040+0000","last_undegraded":"2023-10-09T06:54:46.666040+0000","last_fullsized":"2023-10-09T06:54:46.666040+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455476+0000","last_change":"2023-10-09T06:51:41.233437+0000","last_active":"2023-10-09T06:54:47.455476+0000","last_peered":"2023-10-09T06:54:47.455476+0000","last_clean":"2023-10-09T06:54:47.455476+0000","last_became_active":"2023-10-09T06:51:41.231493+0000","last_became_peered":"2023-10-09T06:51:41.231493+0000","last_unstale":"2023-10-09T06:54:47.455476+0000","last_undegraded":"2023-10-09T06:54:47.455476+0000","last_fullsized":"2023-10-09T06:54:47.455476+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":378,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:48.058470+0000","last_change":"2023-10-09T06:52:29.663419+0000","last_active":"2023-10-09T06:54:48.058470+0000","last_peered":"2023-10-09T06:54:48.058470+0000","last_clean":"2023-10-09T06:54:48.058470+0000","last_became_active":"2023-10-09T06:52:29.663061+0000","last_became_peered":"2023-10-09T06:52:29.663061+0000","last_unstale":"2023-10-09T06:54:48.058470+0000","last_undegraded":"2023-10-09T06:54:48.058470+0000","last_fullsized":"2023-10-09T06:54:48.058470+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:07.866649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669930+0000","last_change":"2023-10-09T06:51:08.233038+0000","last_active":"2023-10-09T06:54:56.669930+0000","last_peered":"2023-10-09T06:54:56.669930+0000","last_clean":"2023-10-09T06:54:56.669930+0000","last_became_active":"2023-10-09T06:51:08.221188+0000","last_became_peered":"2023-10-09T06:51:08.221188+0000","last_unstale":"2023-10-09T06:54:56.669930+0000","last_undegraded":"2023-10-09T06:54:56.669930+0000","last_fullsized":"2023-10-09T06:54:56.669930+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:01.269604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":424,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455901+0000","last_change":"2023-10-09T06:51:08.227179+0000","last_active":"2023-10-09T06:54:47.455901+0000","last_peered":"2023-10-09T06:54:47.455901+0000","last_clean":"2023-10-09T06:54:47.455901+0000","last_became_active":"2023-10-09T06:51:08.226782+0000","last_became_peered":"2023-10-09T06:51:08.226782+0000","last_unstale":"2023-10-09T06:54:47.455901+0000","last_undegraded":"2023-10-09T06:54:47.455901+0000","last_fullsized":"2023-10-09T06:54:47.455901+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666268+0000","last_change":"2023-10-09T06:54:38.034620+0000","last_active":"2023-10-09T06:54:46.666268+0000","last_peered":"2023-10-09T06:54:46.666268+0000","last_clean":"2023-10-09T06:54:46.666268+0000","last_became_active":"2023-10-09T06:54:38.034101+0000","last_became_peered":"2023-10-09T06:54:38.034101+0000","last_unstale":"2023-10-09T06:54:46.666268+0000","last_undegraded":"2023-10-09T06:54:46.666268+0000","last_fullsized":"2023-10-09T06:54:46.666268+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455969+0000","last_change":"2023-10-09T06:51:53.658954+0000","last_active":"2023-10-09T06:54:47.455969+0000","last_peered":"2023-10-09T06:54:47.455969+0000","last_clean":"2023-10-09T06:54:47.455969+0000","last_became_active":"2023-10-09T06:51:53.658393+0000","last_became_peered":"2023-10-09T06:51:53.658393+0000","last_unstale":"2023-10-09T06:54:47.455969+0000","last_undegraded":"2023-10-09T06:54:47.455969+0000","last_fullsized":"2023-10-09T06:54:47.455969+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455157+0000","last_change":"2023-10-09T06:51:53.656986+0000","last_active":"2023-10-09T06:54:47.455157+0000","last_peered":"2023-10-09T06:54:47.455157+0000","last_clean":"2023-10-09T06:54:47.455157+0000","last_became_active":"2023-10-09T06:51:53.656411+0000","last_became_peered":"2023-10-09T06:51:53.656411+0000","last_unstale":"2023-10-09T06:54:47.455157+0000","last_undegraded":"2023-10-09T06:54:47.455157+0000","last_fullsized":"2023-10-09T06:54:47.455157+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":442,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.456560+0000","last_change":"2023-10-09T06:52:41.266484+0000","last_active":"2023-10-09T06:54:47.456560+0000","last_peered":"2023-10-09T06:54:47.456560+0000","last_clean":"2023-10-09T06:54:47.456560+0000","last_became_active":"2023-10-09T06:52:41.261618+0000","last_became_peered":"2023-10-09T06:52:41.261618+0000","last_unstale":"2023-10-09T06:54:47.456560+0000","last_undegraded":"2023-10-09T06:54:47.456560+0000","last_fullsized":"2023-10-09T06:54:47.456560+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669690+0000","last_change":"2023-10-09T06:52:41.249404+0000","last_active":"2023-10-09T06:54:56.669690+0000","last_peered":"2023-10-09T06:54:56.669690+0000","last_clean":"2023-10-09T06:54:56.669690+0000","last_became_active":"2023-10-09T06:52:41.249078+0000","last_became_peered":"2023-10-09T06:52:41.249078+0000","last_unstale":"2023-10-09T06:54:56.669690+0000","last_undegraded":"2023-10-09T06:54:56.669690+0000","last_fullsized":"2023-10-09T06:54:56.669690+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:23.696492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799309+0000","last_change":"2023-10-09T06:52:53.666300+0000","last_active":"2023-10-09T06:54:57.799309+0000","last_peered":"2023-10-09T06:54:57.799309+0000","last_clean":"2023-10-09T06:54:57.799309+0000","last_became_active":"2023-10-09T06:52:53.665861+0000","last_became_peered":"2023-10-09T06:52:53.665861+0000","last_unstale":"2023-10-09T06:54:57.799309+0000","last_undegraded":"2023-10-09T06:54:57.799309+0000","last_fullsized":"2023-10-09T06:54:57.799309+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:02.601488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"350'344","reported_seq":1884,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799503+0000","last_change":"2023-10-09T06:52:53.666148+0000","last_active":"2023-10-09T06:54:57.799503+0000","last_peered":"2023-10-09T06:54:57.799503+0000","last_clean":"2023-10-09T06:54:57.799503+0000","last_became_active":"2023-10-09T06:52:53.665155+0000","last_became_peered":"2023-10-09T06:52:53.665155+0000","last_unstale":"2023-10-09T06:54:57.799503+0000","last_undegraded":"2023-10-09T06:54:57.799503+0000","last_fullsized":"2023-10-09T06:54:57.799503+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":344,"log_dups_size":0,"ondisk_log_size":344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:44.013378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2584,"num_read_kb":4613,"num_write":301,"num_write_kb":2356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":57,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455398+0000","last_change":"2023-10-09T06:51:53.654357+0000","last_active":"2023-10-09T06:54:47.455398+0000","last_peered":"2023-10-09T06:54:47.455398+0000","last_clean":"2023-10-09T06:54:47.455398+0000","last_became_active":"2023-10-09T06:51:53.653956+0000","last_became_peered":"2023-10-09T06:51:53.653956+0000","last_unstale":"2023-10-09T06:54:47.455398+0000","last_undegraded":"2023-10-09T06:54:47.455398+0000","last_fullsized":"2023-10-09T06:54:47.455398+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.798748+0000","last_change":"2023-10-09T06:52:54.125962+0000","last_active":"2023-10-09T06:54:57.798748+0000","last_peered":"2023-10-09T06:54:57.798748+0000","last_clean":"2023-10-09T06:54:57.798748+0000","last_became_active":"2023-10-09T06:52:54.125101+0000","last_became_peered":"2023-10-09T06:52:54.125101+0000","last_unstale":"2023-10-09T06:54:57.798748+0000","last_undegraded":"2023-10-09T06:54:57.798748+0000","last_fullsized":"2023-10-09T06:54:57.798748+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:32.735554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"350'187","reported_seq":697,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455884+0000","last_change":"2023-10-09T06:52:45.244732+0000","last_active":"2023-10-09T06:54:47.455884+0000","last_peered":"2023-10-09T06:54:47.455884+0000","last_clean":"2023-10-09T06:54:47.455884+0000","last_became_active":"2023-10-09T06:52:45.243723+0000","last_became_peered":"2023-10-09T06:52:45.243723+0000","last_unstale":"2023-10-09T06:54:47.455884+0000","last_undegraded":"2023-10-09T06:54:47.455884+0000","last_fullsized":"2023-10-09T06:54:47.455884+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":187,"log_dups_size":0,"ondisk_log_size":187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1616,"num_read_kb":3123,"num_write":176,"num_write_kb":2470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":416,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455744+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:54:47.455744+0000","last_peered":"2023-10-09T06:54:47.455744+0000","last_clean":"2023-10-09T06:54:47.455744+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:54:47.455744+0000","last_undegraded":"2023-10-09T06:54:47.455744+0000","last_fullsized":"2023-10-09T06:54:47.455744+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":60,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799423+0000","last_change":"2023-10-09T06:52:54.126780+0000","last_active":"2023-10-09T06:54:57.799423+0000","last_peered":"2023-10-09T06:54:57.799423+0000","last_clean":"2023-10-09T06:54:57.799423+0000","last_became_active":"2023-10-09T06:52:54.125995+0000","last_became_peered":"2023-10-09T06:52:54.125995+0000","last_unstale":"2023-10-09T06:54:57.799423+0000","last_undegraded":"2023-10-09T06:54:57.799423+0000","last_fullsized":"2023-10-09T06:54:57.799423+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.851665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"350'186","reported_seq":822,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.475930+0000","last_change":"2023-10-09T06:51:56.673791+0000","last_active":"2023-10-09T06:54:57.475930+0000","last_peered":"2023-10-09T06:54:57.475930+0000","last_clean":"2023-10-09T06:54:57.475930+0000","last_became_active":"2023-10-09T06:51:56.672920+0000","last_became_peered":"2023-10-09T06:51:56.672920+0000","last_unstale":"2023-10-09T06:54:57.475930+0000","last_undegraded":"2023-10-09T06:54:57.475930+0000","last_fullsized":"2023-10-09T06:54:57.475930+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":186,"log_dups_size":0,"ondisk_log_size":186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:40.096965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":933,"num_read_kb":1099,"num_write":169,"num_write_kb":4412,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":51,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.510966+0000","last_change":"2023-10-09T06:52:54.126091+0000","last_active":"2023-10-09T06:54:56.510966+0000","last_peered":"2023-10-09T06:54:56.510966+0000","last_clean":"2023-10-09T06:54:56.510966+0000","last_became_active":"2023-10-09T06:52:54.125482+0000","last_became_peered":"2023-10-09T06:52:54.125482+0000","last_unstale":"2023-10-09T06:54:56.510966+0000","last_undegraded":"2023-10-09T06:54:56.510966+0000","last_fullsized":"2023-10-09T06:54:56.510966+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:40.982672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799213+0000","last_change":"2023-10-09T06:52:54.125788+0000","last_active":"2023-10-09T06:54:57.799213+0000","last_peered":"2023-10-09T06:54:57.799213+0000","last_clean":"2023-10-09T06:54:57.799213+0000","last_became_active":"2023-10-09T06:52:54.124991+0000","last_became_peered":"2023-10-09T06:52:54.124991+0000","last_unstale":"2023-10-09T06:54:57.799213+0000","last_undegraded":"2023-10-09T06:54:57.799213+0000","last_fullsized":"2023-10-09T06:54:57.799213+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:11.624260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"350'307","reported_seq":1097,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455252+0000","last_change":"2023-10-09T06:51:58.697612+0000","last_active":"2023-10-09T06:54:47.455252+0000","last_peered":"2023-10-09T06:54:47.455252+0000","last_clean":"2023-10-09T06:54:47.455252+0000","last_became_active":"2023-10-09T06:51:58.696659+0000","last_became_peered":"2023-10-09T06:51:58.696659+0000","last_unstale":"2023-10-09T06:54:47.455252+0000","last_undegraded":"2023-10-09T06:54:47.455252+0000","last_fullsized":"2023-10-09T06:54:47.455252+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":307,"log_dups_size":0,"ondisk_log_size":307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":752,"num_read_kb":4990,"num_write":302,"num_write_kb":3916,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":700,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180621+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:54:47.180621+0000","last_peered":"2023-10-09T06:54:47.180621+0000","last_clean":"2023-10-09T06:54:47.180621+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:54:47.180621+0000","last_undegraded":"2023-10-09T06:54:47.180621+0000","last_fullsized":"2023-10-09T06:54:47.180621+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669912+0000","last_change":"2023-10-09T06:51:41.233039+0000","last_active":"2023-10-09T06:54:56.669912+0000","last_peered":"2023-10-09T06:54:56.669912+0000","last_clean":"2023-10-09T06:54:56.669912+0000","last_became_active":"2023-10-09T06:51:41.232622+0000","last_became_peered":"2023-10-09T06:51:41.232622+0000","last_unstale":"2023-10-09T06:54:56.669912+0000","last_undegraded":"2023-10-09T06:54:56.669912+0000","last_fullsized":"2023-10-09T06:54:56.669912+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:40.553458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666349+0000","last_change":"2023-10-09T06:54:38.035139+0000","last_active":"2023-10-09T06:54:46.666349+0000","last_peered":"2023-10-09T06:54:46.666349+0000","last_clean":"2023-10-09T06:54:46.666349+0000","last_became_active":"2023-10-09T06:54:38.034786+0000","last_became_peered":"2023-10-09T06:54:38.034786+0000","last_unstale":"2023-10-09T06:54:46.666349+0000","last_undegraded":"2023-10-09T06:54:46.666349+0000","last_fullsized":"2023-10-09T06:54:46.666349+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"350'198","reported_seq":944,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181106+0000","last_change":"2023-10-09T06:52:46.247372+0000","last_active":"2023-10-09T06:54:47.181106+0000","last_peered":"2023-10-09T06:54:47.181106+0000","last_clean":"2023-10-09T06:54:47.181106+0000","last_became_active":"2023-10-09T06:52:46.246564+0000","last_became_peered":"2023-10-09T06:52:46.246564+0000","last_unstale":"2023-10-09T06:54:47.181106+0000","last_undegraded":"2023-10-09T06:54:47.181106+0000","last_fullsized":"2023-10-09T06:54:47.181106+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":198,"log_dups_size":0,"ondisk_log_size":198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1395,"num_read_kb":2773,"num_write":167,"num_write_kb":1385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":277,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450224+0000","last_change":"2023-10-09T06:52:29.659138+0000","last_active":"2023-10-09T06:54:47.450224+0000","last_peered":"2023-10-09T06:54:47.450224+0000","last_clean":"2023-10-09T06:54:47.450224+0000","last_became_active":"2023-10-09T06:52:29.658791+0000","last_became_peered":"2023-10-09T06:52:29.658791+0000","last_unstale":"2023-10-09T06:54:47.450224+0000","last_undegraded":"2023-10-09T06:54:47.450224+0000","last_fullsized":"2023-10-09T06:54:47.450224+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:52.081375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":411,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450302+0000","last_change":"2023-10-09T06:52:41.248968+0000","last_active":"2023-10-09T06:54:47.450302+0000","last_peered":"2023-10-09T06:54:47.450302+0000","last_clean":"2023-10-09T06:54:47.450302+0000","last_became_active":"2023-10-09T06:52:41.248572+0000","last_became_peered":"2023-10-09T06:52:41.248572+0000","last_unstale":"2023-10-09T06:54:47.450302+0000","last_undegraded":"2023-10-09T06:54:47.450302+0000","last_fullsized":"2023-10-09T06:54:47.450302+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:38.052232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"350'194","reported_seq":748,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180869+0000","last_change":"2023-10-09T06:50:42.202899+0000","last_active":"2023-10-09T06:54:47.180869+0000","last_peered":"2023-10-09T06:54:47.180869+0000","last_clean":"2023-10-09T06:54:47.180869+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:54:47.180869+0000","last_undegraded":"2023-10-09T06:54:47.180869+0000","last_fullsized":"2023-10-09T06:54:47.180869+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1332,"num_read_kb":3158,"num_write":169,"num_write_kb":2120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799598+0000","last_change":"2023-10-09T06:52:54.126837+0000","last_active":"2023-10-09T06:54:57.799598+0000","last_peered":"2023-10-09T06:54:57.799598+0000","last_clean":"2023-10-09T06:54:57.799598+0000","last_became_active":"2023-10-09T06:52:54.126361+0000","last_became_peered":"2023-10-09T06:52:54.126361+0000","last_unstale":"2023-10-09T06:54:57.799598+0000","last_undegraded":"2023-10-09T06:54:57.799598+0000","last_fullsized":"2023-10-09T06:54:57.799598+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:44.013804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.452498+0000","last_change":"2023-10-09T06:52:29.660430+0000","last_active":"2023-10-09T06:54:47.452498+0000","last_peered":"2023-10-09T06:54:47.452498+0000","last_clean":"2023-10-09T06:54:47.452498+0000","last_became_active":"2023-10-09T06:52:29.660175+0000","last_became_peered":"2023-10-09T06:52:29.660175+0000","last_unstale":"2023-10-09T06:54:47.452498+0000","last_undegraded":"2023-10-09T06:54:47.452498+0000","last_fullsized":"2023-10-09T06:54:47.452498+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:27.353492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670469+0000","last_change":"2023-10-09T06:51:08.224948+0000","last_active":"2023-10-09T06:54:56.670469+0000","last_peered":"2023-10-09T06:54:56.670469+0000","last_clean":"2023-10-09T06:54:56.670469+0000","last_became_active":"2023-10-09T06:51:08.212213+0000","last_became_peered":"2023-10-09T06:51:08.212213+0000","last_unstale":"2023-10-09T06:54:56.670469+0000","last_undegraded":"2023-10-09T06:54:56.670469+0000","last_fullsized":"2023-10-09T06:54:56.670469+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:55.420006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"350'241","reported_seq":1306,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799369+0000","last_change":"2023-10-09T06:52:53.667462+0000","last_active":"2023-10-09T06:54:57.799369+0000","last_peered":"2023-10-09T06:54:57.799369+0000","last_clean":"2023-10-09T06:54:57.799369+0000","last_became_active":"2023-10-09T06:52:53.666474+0000","last_became_peered":"2023-10-09T06:52:53.666474+0000","last_unstale":"2023-10-09T06:54:57.799369+0000","last_undegraded":"2023-10-09T06:54:57.799369+0000","last_fullsized":"2023-10-09T06:54:57.799369+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":241,"log_dups_size":0,"ondisk_log_size":241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:53.594480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1357,"num_read_kb":2514,"num_write":167,"num_write_kb":5394,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.475823+0000","last_change":"2023-10-09T06:52:21.261388+0000","last_active":"2023-10-09T06:54:57.475823+0000","last_peered":"2023-10-09T06:54:57.475823+0000","last_clean":"2023-10-09T06:54:57.475823+0000","last_became_active":"2023-10-09T06:52:21.260902+0000","last_became_peered":"2023-10-09T06:52:21.260902+0000","last_unstale":"2023-10-09T06:54:57.475823+0000","last_undegraded":"2023-10-09T06:54:57.475823+0000","last_fullsized":"2023-10-09T06:54:57.475823+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:09.200069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"350'293","reported_seq":1503,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666529+0000","last_change":"2023-10-09T06:51:56.678019+0000","last_active":"2023-10-09T06:54:46.666529+0000","last_peered":"2023-10-09T06:54:46.666529+0000","last_clean":"2023-10-09T06:54:46.666529+0000","last_became_active":"2023-10-09T06:51:56.677244+0000","last_became_peered":"2023-10-09T06:51:56.677244+0000","last_unstale":"2023-10-09T06:54:46.666529+0000","last_undegraded":"2023-10-09T06:54:46.666529+0000","last_fullsized":"2023-10-09T06:54:46.666529+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1929,"num_read_kb":3111,"num_write":239,"num_write_kb":3373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":306,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450522+0000","last_change":"2023-10-09T06:52:41.278834+0000","last_active":"2023-10-09T06:54:47.450522+0000","last_peered":"2023-10-09T06:54:47.450522+0000","last_clean":"2023-10-09T06:54:47.450522+0000","last_became_active":"2023-10-09T06:52:41.259514+0000","last_became_peered":"2023-10-09T06:52:41.259514+0000","last_unstale":"2023-10-09T06:54:47.450522+0000","last_undegraded":"2023-10-09T06:54:47.450522+0000","last_fullsized":"2023-10-09T06:54:47.450522+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:32.900490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":409,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455255+0000","last_change":"2023-10-09T06:51:41.229292+0000","last_active":"2023-10-09T06:54:47.455255+0000","last_peered":"2023-10-09T06:54:47.455255+0000","last_clean":"2023-10-09T06:54:47.455255+0000","last_became_active":"2023-10-09T06:51:41.228949+0000","last_became_peered":"2023-10-09T06:51:41.228949+0000","last_unstale":"2023-10-09T06:54:47.455255+0000","last_undegraded":"2023-10-09T06:54:47.455255+0000","last_fullsized":"2023-10-09T06:54:47.455255+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123779+0000","last_change":"2023-10-09T06:52:29.660475+0000","last_active":"2023-10-09T06:54:57.123779+0000","last_peered":"2023-10-09T06:54:57.123779+0000","last_clean":"2023-10-09T06:54:57.123779+0000","last_became_active":"2023-10-09T06:52:29.660091+0000","last_became_peered":"2023-10-09T06:52:29.660091+0000","last_unstale":"2023-10-09T06:54:57.123779+0000","last_undegraded":"2023-10-09T06:54:57.123779+0000","last_fullsized":"2023-10-09T06:54:57.123779+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:29.940623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":45,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450787+0000","last_change":"2023-10-09T06:51:53.650329+0000","last_active":"2023-10-09T06:54:47.450787+0000","last_peered":"2023-10-09T06:54:47.450787+0000","last_clean":"2023-10-09T06:54:47.450787+0000","last_became_active":"2023-10-09T06:51:53.649959+0000","last_became_peered":"2023-10-09T06:51:53.649959+0000","last_unstale":"2023-10-09T06:54:47.450787+0000","last_undegraded":"2023-10-09T06:54:47.450787+0000","last_fullsized":"2023-10-09T06:54:47.450787+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:08.498803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":65,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455297+0000","last_change":"2023-10-09T06:52:41.327440+0000","last_active":"2023-10-09T06:54:47.455297+0000","last_peered":"2023-10-09T06:54:47.455297+0000","last_clean":"2023-10-09T06:54:47.455297+0000","last_became_active":"2023-10-09T06:52:41.266476+0000","last_became_peered":"2023-10-09T06:52:41.266476+0000","last_unstale":"2023-10-09T06:54:47.455297+0000","last_undegraded":"2023-10-09T06:54:47.455297+0000","last_fullsized":"2023-10-09T06:54:47.455297+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455031+0000","last_change":"2023-10-09T06:52:41.266379+0000","last_active":"2023-10-09T06:54:47.455031+0000","last_peered":"2023-10-09T06:54:47.455031+0000","last_clean":"2023-10-09T06:54:47.455031+0000","last_became_active":"2023-10-09T06:52:41.262339+0000","last_became_peered":"2023-10-09T06:52:41.262339+0000","last_unstale":"2023-10-09T06:54:47.455031+0000","last_undegraded":"2023-10-09T06:54:47.455031+0000","last_fullsized":"2023-10-09T06:54:47.455031+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":52,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455551+0000","last_change":"2023-10-09T06:51:41.326746+0000","last_active":"2023-10-09T06:54:47.455551+0000","last_peered":"2023-10-09T06:54:47.455551+0000","last_clean":"2023-10-09T06:54:47.455551+0000","last_became_active":"2023-10-09T06:51:41.235036+0000","last_became_peered":"2023-10-09T06:51:41.235036+0000","last_unstale":"2023-10-09T06:54:47.455551+0000","last_undegraded":"2023-10-09T06:54:47.455551+0000","last_fullsized":"2023-10-09T06:54:47.455551+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455367+0000","last_change":"2023-10-09T06:51:08.218437+0000","last_active":"2023-10-09T06:54:47.455367+0000","last_peered":"2023-10-09T06:54:47.455367+0000","last_clean":"2023-10-09T06:54:47.455367+0000","last_became_active":"2023-10-09T06:51:08.218044+0000","last_became_peered":"2023-10-09T06:51:08.218044+0000","last_unstale":"2023-10-09T06:54:47.455367+0000","last_undegraded":"2023-10-09T06:54:47.455367+0000","last_fullsized":"2023-10-09T06:54:47.455367+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670509+0000","last_change":"2023-10-09T06:51:53.658314+0000","last_active":"2023-10-09T06:54:56.670509+0000","last_peered":"2023-10-09T06:54:56.670509+0000","last_clean":"2023-10-09T06:54:56.670509+0000","last_became_active":"2023-10-09T06:51:53.656771+0000","last_became_peered":"2023-10-09T06:51:53.656771+0000","last_unstale":"2023-10-09T06:54:56.670509+0000","last_undegraded":"2023-10-09T06:54:56.670509+0000","last_fullsized":"2023-10-09T06:54:56.670509+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:21.614043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669671+0000","last_change":"2023-10-09T06:51:53.657581+0000","last_active":"2023-10-09T06:54:56.669671+0000","last_peered":"2023-10-09T06:54:56.669671+0000","last_clean":"2023-10-09T06:54:56.669671+0000","last_became_active":"2023-10-09T06:51:53.656320+0000","last_became_peered":"2023-10-09T06:51:53.656320+0000","last_unstale":"2023-10-09T06:54:56.669671+0000","last_undegraded":"2023-10-09T06:54:56.669671+0000","last_fullsized":"2023-10-09T06:54:56.669671+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:42.694344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":361,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450952+0000","last_change":"2023-10-09T06:52:29.658771+0000","last_active":"2023-10-09T06:54:47.450952+0000","last_peered":"2023-10-09T06:54:47.450952+0000","last_clean":"2023-10-09T06:54:47.450952+0000","last_became_active":"2023-10-09T06:52:29.658306+0000","last_became_peered":"2023-10-09T06:52:29.658306+0000","last_unstale":"2023-10-09T06:54:47.450952+0000","last_undegraded":"2023-10-09T06:54:47.450952+0000","last_fullsized":"2023-10-09T06:54:47.450952+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:20.760541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":460,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.454948+0000","last_change":"2023-10-09T06:52:41.266360+0000","last_active":"2023-10-09T06:54:47.454948+0000","last_peered":"2023-10-09T06:54:47.454948+0000","last_clean":"2023-10-09T06:54:47.454948+0000","last_became_active":"2023-10-09T06:52:41.261562+0000","last_became_peered":"2023-10-09T06:52:41.261562+0000","last_unstale":"2023-10-09T06:54:47.454948+0000","last_undegraded":"2023-10-09T06:54:47.454948+0000","last_fullsized":"2023-10-09T06:54:47.454948+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":31644,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11898,"num_read_kb":25381,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":16777368,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430903,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910212,"kb_used_data":4092,"kb_used_omap":0,"kb_used_meta":906112,"kb_avail":92834940,"statfs":{"total":95995035648,"available":95062978560,"internally_reserved":0,"allocated":4190208,"data_stored":3510378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927858688},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.879,"15min":1.3560000000000001},"min":{"1min":0.56100000000000005,"5min":0.40899999999999997,"15min":0.224},"max":{"1min":1.28,"5min":1.3280000000000001,"15min":92.066999999999993},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.88100000000000001,"15min":1.377},"min":{"1min":0.66900000000000004,"5min":0.47199999999999998,"15min":0.27600000000000002},"max":{"1min":1.2350000000000001,"5min":1.4219999999999999,"15min":91.715999999999994},"last":0.628}]},{"osd":1,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92200000000000004,"15min":1.2210000000000001},"min":{"1min":0.58499999999999996,"5min":0.45800000000000002,"15min":0.28699999999999998},"max":{"1min":1.3220000000000001,"5min":1.6299999999999999,"15min":13.526999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.97999999999999998,"15min":1.1200000000000001},"min":{"1min":0.67100000000000004,"5min":0.39400000000000002,"15min":0.27200000000000002},"max":{"1min":1.347,"5min":1.5149999999999999,"15min":10.798999999999999},"last":0.72899999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.89300000000000002,"15min":1.329},"min":{"1min":0.63100000000000001,"5min":0.61699999999999999,"15min":0.29099999999999998},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":9.0679999999999996},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0009999999999999,"15min":1.1399999999999999},"min":{"1min":0.748,"5min":0.65000000000000002,"15min":0.34699999999999998},"max":{"1min":1.2889999999999999,"5min":1.4710000000000001,"15min":6.3319999999999999},"last":0.89600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.98799999999999999,"15min":1.649},"min":{"1min":0.69299999999999995,"5min":0.61899999999999999,"15min":0.36499999999999999},"max":{"1min":1.111,"5min":1.4079999999999999,"15min":23.074000000000002},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.119,"5min":1.0980000000000001,"15min":1.425},"min":{"1min":0.79100000000000004,"5min":0.73499999999999999,"15min":0.36699999999999999},"max":{"1min":1.4139999999999999,"5min":1.677,"15min":23.163},"last":0.97699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.94099999999999995,"15min":1.1519999999999999},"min":{"1min":0.56999999999999995,"5min":0.43099999999999999,"15min":0.26700000000000002},"max":{"1min":1.149,"5min":1.4670000000000001,"15min":12.711},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.96899999999999997,"15min":1.5800000000000001},"min":{"1min":0.64600000000000002,"5min":0.53900000000000003,"15min":0.32700000000000001},"max":{"1min":1.2090000000000001,"5min":1.484,"15min":11.369999999999999},"last":1.0029999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0820000000000001,"15min":1.107},"min":{"1min":0.59199999999999997,"5min":0.53300000000000003,"15min":0.35099999999999998},"max":{"1min":1.4410000000000001,"5min":1.6240000000000001,"15min":6.3049999999999997},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.069,"15min":1.0580000000000001},"min":{"1min":0.64400000000000002,"5min":0.61399999999999999,"15min":0.34100000000000003},"max":{"1min":1.3260000000000001,"5min":1.4650000000000001,"15min":12.675000000000001},"last":0.872}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.095,"15min":1.3480000000000001},"min":{"1min":0.61199999999999999,"5min":0.56499999999999995,"15min":0.32200000000000001},"max":{"1min":1.3600000000000001,"5min":1.5349999999999999,"15min":12.640000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.095,"5min":1.127,"15min":1.6739999999999999},"min":{"1min":0.629,"5min":0.59899999999999998,"15min":0.40500000000000003},"max":{"1min":1.3049999999999999,"5min":1.611,"15min":14.613},"last":0.78500000000000003}]}]},{"osd":6,"up_from":42,"seq":180388627135,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900596,"kb_used_data":8172,"kb_used_omap":0,"kb_used_meta":892416,"kb_avail":92844556,"statfs":{"total":95995035648,"available":95072825344,"internally_reserved":0,"allocated":8368128,"data_stored":7703925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913833984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.89400000000000002,"15min":2.1739999999999999},"min":{"1min":0.65100000000000002,"5min":0.34799999999999998,"15min":0.23799999999999999},"max":{"1min":1.264,"5min":1.5720000000000001,"15min":120.822},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.90600000000000003,"15min":2.3159999999999998},"min":{"1min":0.627,"5min":0.375,"15min":0.28899999999999998},"max":{"1min":1.1679999999999999,"5min":1.381,"15min":120.47199999999999},"last":1.1639999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.873,"15min":1.1850000000000001},"min":{"1min":0.60499999999999998,"5min":0.53000000000000003,"15min":0.33300000000000002},"max":{"1min":1.1819999999999999,"5min":1.464,"15min":7.7539999999999996},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.95299999999999996,"15min":1.1579999999999999},"min":{"1min":0.65800000000000003,"5min":0.44800000000000001,"15min":0.29999999999999999},"max":{"1min":1.2909999999999999,"5min":1.4379999999999999,"15min":7.04},"last":0.84499999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.94599999999999995,"15min":2.2879999999999998},"min":{"1min":0.58299999999999996,"5min":0.56799999999999995,"15min":0.30599999999999999},"max":{"1min":1.242,"5min":1.3440000000000001,"15min":271.976},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.94299999999999995,"15min":2.3900000000000001},"min":{"1min":0.66700000000000004,"5min":0.58499999999999996,"15min":0.316},"max":{"1min":1.2150000000000001,"5min":1.353,"15min":271.863},"last":0.96099999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99299999999999999,"15min":1.2929999999999999},"min":{"1min":0.629,"5min":0.57599999999999996,"15min":0.35799999999999998},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":22.873999999999999},"last":1.089},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.98399999999999999,"15min":1.486},"min":{"1min":0.68500000000000005,"5min":0.63800000000000001,"15min":0.35099999999999998},"max":{"1min":1.198,"5min":3.0209999999999999,"15min":25.986999999999998},"last":0.89500000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.93000000000000005,"15min":1.77},"min":{"1min":0.60699999999999998,"5min":0.42999999999999999,"15min":0.30199999999999999},"max":{"1min":1.278,"5min":1.3859999999999999,"15min":80.231999999999999},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.97399999999999998,"15min":1.6779999999999999},"min":{"1min":0.63500000000000001,"5min":0.504,"15min":0.28799999999999998},"max":{"1min":1.246,"5min":1.548,"15min":81.085999999999999},"last":1.127}]},{"osd":5,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0529999999999999,"15min":1.218},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.34899999999999998},"max":{"1min":1.3340000000000001,"5min":1.4510000000000001,"15min":11.558},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0449999999999999,"15min":1.1359999999999999},"min":{"1min":0.72699999999999998,"5min":0.61899999999999999,"15min":0.311},"max":{"1min":1.292,"5min":1.643,"15min":5.7400000000000002},"last":0.70799999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.1000000000000001,"15min":1.619},"min":{"1min":0.82999999999999996,"5min":0.58499999999999996,"15min":0.36099999999999999},"max":{"1min":1.2769999999999999,"5min":1.5629999999999999,"15min":12.177},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.143,"5min":1.1359999999999999,"15min":1.2789999999999999},"min":{"1min":0.60199999999999998,"5min":0.41699999999999998,"15min":0.35399999999999998},"max":{"1min":1.347,"5min":1.5389999999999999,"15min":9.5619999999999994},"last":1.1950000000000001}]}]},{"osd":1,"up_from":93,"seq":399431959045,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":683572,"kb_used_data":8428,"kb_used_omap":7,"kb_used_meta":675128,"kb_avail":93061580,"statfs":{"total":95995035648,"available":95295057920,"internally_reserved":0,"allocated":8630272,"data_stored":7208150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":691331472},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.72999999999999998,"15min":1.0309999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.19600000000000001},"max":{"1min":1.028,"5min":5.3230000000000004,"15min":17.201000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.73199999999999998,"15min":0.97099999999999997},"min":{"1min":0.41499999999999998,"5min":0.29199999999999998,"15min":0.17899999999999999},"max":{"1min":0.94799999999999995,"5min":2.169,"15min":15.981999999999999},"last":0.81000000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.71799999999999997,"15min":1.111},"min":{"1min":0.36799999999999999,"5min":0.26500000000000001,"15min":0.20999999999999999},"max":{"1min":1.133,"5min":2.0760000000000001,"15min":16.004000000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.82099999999999995,"15min":1.2749999999999999},"min":{"1min":0.45100000000000001,"5min":0.38900000000000001,"15min":0.22600000000000001},"max":{"1min":1.0589999999999999,"5min":1.331,"15min":16.971},"last":1.1000000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.90100000000000002,"15min":1.4430000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.27400000000000002},"max":{"1min":1.2729999999999999,"5min":1.986,"15min":47.143000000000001},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.83599999999999997,"15min":1.2450000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.27500000000000002},"max":{"1min":1.0309999999999999,"5min":5.4260000000000002,"15min":46.826999999999998},"last":0.95699999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.083,"15min":1.4279999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.318},"max":{"1min":1.4910000000000001,"5min":1.863,"15min":28.626999999999999},"last":1.29},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.026,"15min":1.262},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.317},"max":{"1min":1.1259999999999999,"5min":5.5949999999999998,"15min":30},"last":1.0129999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.038,"15min":1.012},"min":{"1min":0.66600000000000004,"5min":0.66000000000000003,"15min":0.38700000000000001},"max":{"1min":1.3360000000000001,"5min":4.3579999999999997,"15min":4.9000000000000004},"last":1.3879999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1599999999999999,"15min":1.1830000000000001},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.375},"max":{"1min":1.5389999999999999,"5min":4.2350000000000003,"15min":8.077},"last":1.579}]},{"osd":6,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.081,"15min":1.2130000000000001},"min":{"1min":0.70599999999999996,"5min":0.67100000000000004,"15min":0.36899999999999999},"max":{"1min":1.431,"5min":4.5419999999999998,"15min":25.553000000000001},"last":1.3280000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.1879999999999999,"15min":1.4370000000000001},"min":{"1min":0.748,"5min":0.66600000000000004,"15min":0.40100000000000002},"max":{"1min":1.544,"5min":4.1340000000000003,"15min":25.675000000000001},"last":1.407}]},{"osd":7,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.216,"15min":1.4730000000000001},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.41899999999999998},"max":{"1min":1.4730000000000001,"5min":3.8759999999999999,"15min":25.585999999999999},"last":1.5009999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.1599999999999999,"15min":1.353},"min":{"1min":0.81799999999999995,"5min":0.69899999999999995,"15min":0.432},"max":{"1min":1.355,"5min":5.718,"15min":25.518000000000001},"last":1.2549999999999999}]}]},{"osd":0,"up_from":88,"seq":377957122570,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":940480,"kb_used_data":8952,"kb_used_omap":8,"kb_used_meta":931511,"kb_avail":92804672,"statfs":{"total":95995035648,"available":95031984128,"internally_reserved":0,"allocated":9166848,"data_stored":7789639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":953867520},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:56:00 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.73399999999999999,"15min":1.026},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.20000000000000001},"max":{"1min":1.0680000000000001,"5min":1.921,"15min":10.449},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.82699999999999996,"15min":1.0800000000000001},"min":{"1min":0.57199999999999995,"5min":0.375,"15min":0.254},"max":{"1min":1.1659999999999999,"5min":1.3839999999999999,"15min":13.055999999999999},"last":1.1930000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.72199999999999998,"15min":1.036},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.20599999999999999},"max":{"1min":0.78500000000000003,"5min":1.9810000000000001,"15min":12.154},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.70299999999999996,"15min":0.96699999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.223},"max":{"1min":0.879,"5min":1.1759999999999999,"15min":12.448},"last":0.83399999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:56:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.70399999999999996,"15min":1.0509999999999999},"min":{"1min":0.42999999999999999,"5min":0.28799999999999998,"15min":0.23300000000000001},"max":{"1min":0.95699999999999996,"5min":1.2290000000000001,"15min":11.635999999999999},"last":0.878},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.83699999999999997,"15min":1.224},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.25800000000000001},"max":{"1min":0.95499999999999996,"5min":1.425,"15min":15.545999999999999},"last":1.0960000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93000000000000005,"15min":1.2210000000000001},"min":{"1min":0.65300000000000002,"5min":0.57199999999999995,"15min":0.32800000000000001},"max":{"1min":1.2949999999999999,"5min":1.4790000000000001,"15min":9.5459999999999994},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.999,"5min":0.98899999999999999,"15min":1.22},"min":{"1min":0.59199999999999997,"5min":0.501,"15min":0.31},"max":{"1min":1.3540000000000001,"5min":2.056,"15min":11.948},"last":1.135}]},{"osd":5,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.016,"15min":1.1259999999999999},"min":{"1min":0.755,"5min":0.60499999999999998,"15min":0.36699999999999999},"max":{"1min":1.3009999999999999,"5min":2.02,"15min":11.894},"last":1.224},{"interface":"front","average":{"1min":1.077,"5min":1.095,"15min":1.1779999999999999},"min":{"1min":0.63,"5min":0.47799999999999998,"15min":0.40400000000000003},"max":{"1min":1.3300000000000001,"5min":1.6200000000000001,"15min":8.5630000000000006},"last":1.3049999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":1.109,"15min":1.379},"min":{"1min":0.85099999999999998,"5min":0.53700000000000003,"15min":0.36799999999999999},"max":{"1min":1.3939999999999999,"5min":1.5780000000000001,"15min":8.5269999999999992},"last":1.3640000000000001},{"interface":"front","average":{"1min":1.069,"5min":1.0489999999999999,"15min":1.27},"min":{"1min":0.70599999999999996,"5min":0.51400000000000001,"15min":0.379},"max":{"1min":1.3220000000000001,"5min":1.764,"15min":6.5869999999999997},"last":1.165}]},{"osd":7,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1759999999999999,"15min":1.4450000000000001},"min":{"1min":0.875,"5min":0.64900000000000002,"15min":0.40100000000000002},"max":{"1min":1.4930000000000001,"5min":1.6839999999999999,"15min":8.4819999999999993},"last":1.466},{"interface":"front","average":{"1min":1.125,"5min":1.089,"15min":1.3640000000000001},"min":{"1min":0.65200000000000002,"5min":0.46000000000000002,"15min":0.39300000000000002},"max":{"1min":1.3460000000000001,"5min":1.659,"15min":6.6120000000000001},"last":1.3320000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795519,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670316,"kb_used_data":8420,"kb_used_omap":11,"kb_used_meta":661876,"kb_avail":93074836,"statfs":{"total":95995035648,"available":95308632064,"internally_reserved":0,"allocated":8622080,"data_stored":7202150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":677761982},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.68200000000000005,"15min":1.111},"min":{"1min":0.311,"5min":0.311,"15min":0.185},"max":{"1min":0.86299999999999999,"5min":1.228,"15min":38.210000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":1.0509999999999999},"min":{"1min":0.433,"5min":0.40200000000000002,"15min":0.219},"max":{"1min":1.244,"5min":1.244,"15min":39.896000000000001},"last":0.497}]},{"osd":1,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.74199999999999999,"15min":0.92100000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.223},"max":{"1min":1.024,"5min":1.292,"15min":11.757},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.75600000000000001,"15min":1.0429999999999999},"min":{"1min":0.39000000000000001,"5min":0.36199999999999999,"15min":0.223},"max":{"1min":0.95399999999999996,"5min":1.306,"15min":42.710000000000001},"last":0.76800000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.75800000000000001,"15min":1.575},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.24399999999999999},"max":{"1min":0.96699999999999997,"5min":1.196,"15min":52.060000000000002},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.747,"5min":0.74299999999999999,"15min":1.502},"min":{"1min":0.44700000000000001,"5min":0.37,"15min":0.255},"max":{"1min":1.1279999999999999,"5min":1.349,"15min":51.880000000000003},"last":0.95999999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.999,"15min":1.3089999999999999},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.33700000000000002},"max":{"1min":1.381,"5min":4.1920000000000002,"15min":43.244},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.99299999999999999,"15min":1.3},"min":{"1min":0.66700000000000004,"5min":0.65200000000000002,"15min":0.32100000000000001},"max":{"1min":1.085,"5min":4.2800000000000002,"15min":42.783999999999999},"last":1.0600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.028,"15min":1.111},"min":{"1min":0.63400000000000001,"5min":0.57999999999999996,"15min":0.35999999999999999},"max":{"1min":1.3089999999999999,"5min":3.4039999999999999,"15min":43.210000000000001},"last":1.135},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.034,"15min":1.196},"min":{"1min":0.69699999999999995,"5min":0.52800000000000002,"15min":0.371},"max":{"1min":1.403,"5min":3.601,"15min":42.737000000000002},"last":1.3520000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.042,"15min":1.296},"min":{"1min":0.749,"5min":0.68400000000000005,"15min":0.36199999999999999},"max":{"1min":1.2390000000000001,"5min":2.968,"15min":16.332999999999998},"last":1.2969999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.113,"15min":1.379},"min":{"1min":0.68799999999999994,"5min":0.65500000000000003,"15min":0.35799999999999998},"max":{"1min":1.2889999999999999,"5min":3.7989999999999999,"15min":42.834000000000003},"last":1.177}]},{"osd":7,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.155,"15min":1.5369999999999999},"min":{"1min":0.84199999999999997,"5min":0.61899999999999999,"15min":0.39000000000000001},"max":{"1min":1.353,"5min":3.2879999999999998,"15min":42.804000000000002},"last":1.266},{"interface":"front","average":{"1min":1.048,"5min":1.141,"15min":1.5569999999999999},"min":{"1min":0.80400000000000005,"5min":0.51300000000000001,"15min":0.40200000000000002},"max":{"1min":1.429,"5min":3.3370000000000002,"15min":43.164000000000001},"last":1.24}]}]},{"osd":3,"up_from":103,"seq":442381631993,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757920,"kb_used_data":8472,"kb_used_omap":3,"kb_used_meta":749436,"kb_avail":92987232,"statfs":{"total":95995035648,"available":95218925568,"internally_reserved":0,"allocated":8675328,"data_stored":7206641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":767423009},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.70999999999999996,"15min":1.131},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.20699999999999999},"max":{"1min":1.0640000000000001,"5min":1.2470000000000001,"15min":44.442},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.70099999999999996,"15min":1.181},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.218},"max":{"1min":0.86599999999999999,"5min":1.226,"15min":44.250999999999998},"last":0.73299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.69499999999999995,"15min":1.0649999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.221},"max":{"1min":0.85599999999999998,"5min":1.0600000000000001,"15min":11.577999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.75900000000000001,"15min":1.0820000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.21299999999999999},"max":{"1min":1.0149999999999999,"5min":1.1140000000000001,"15min":10.305999999999999},"last":0.86699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":1.5669999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.25700000000000001},"max":{"1min":1.238,"5min":1.238,"15min":166.89500000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.79200000000000004,"15min":1.542},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.28100000000000003},"max":{"1min":1.278,"5min":1.3220000000000001,"15min":165.86600000000001},"last":0.93600000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.94799999999999995,"15min":1.208},"min":{"1min":0.59799999999999998,"5min":0.46600000000000003,"15min":0.30299999999999999},"max":{"1min":1.351,"5min":1.48,"15min":11.673999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.01,"5min":0.99399999999999999,"15min":1.254},"min":{"1min":0.68799999999999994,"5min":0.54000000000000004,"15min":0.32800000000000001},"max":{"1min":1.429,"5min":1.5529999999999999,"15min":11.365},"last":1.016}]},{"osd":5,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.2150000000000001},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.35799999999999998},"max":{"1min":1.4850000000000001,"5min":1.5109999999999999,"15min":11.439},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.048,"15min":1.2170000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.33700000000000002},"max":{"1min":1.3959999999999999,"5min":1.528,"15min":11.763},"last":1.3899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.1120000000000001,"15min":1.1910000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.379},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":9.718},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.117,"15min":1.3859999999999999},"min":{"1min":0.73299999999999998,"5min":0.71299999999999997,"15min":0.39600000000000002},"max":{"1min":1.381,"5min":1.6000000000000001,"15min":15.233000000000001},"last":1.1419999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.1040000000000001,"15min":1.4079999999999999},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.35899999999999999},"max":{"1min":1.468,"5min":1.5920000000000001,"15min":11.731},"last":1.3140000000000001},{"interface":"front","average":{"1min":1.143,"5min":1.155,"15min":1.4279999999999999},"min":{"1min":0.67200000000000004,"5min":0.65000000000000002,"15min":0.35799999999999998},"max":{"1min":1.536,"5min":1.536,"15min":10.843},"last":1.1850000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019601,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906876,"kb_used_data":7604,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92838276,"statfs":{"total":95995035648,"available":95066394624,"internally_reserved":0,"allocated":7786496,"data_stored":7114244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:37 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.96399999999999997,"15min":2.3530000000000002},"min":{"1min":0.68200000000000005,"5min":0.46300000000000002,"15min":0.28199999999999997},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":91.683999999999997},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.04,"5min":0.996,"15min":2.4529999999999998},"min":{"1min":0.753,"5min":0.52600000000000002,"15min":0.26000000000000001},"max":{"1min":1.2969999999999999,"5min":1.381,"15min":91.653999999999996},"last":1.1970000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.96999999999999997,"15min":1.139},"min":{"1min":0.72799999999999998,"5min":0.59899999999999998,"15min":0.28499999999999998},"max":{"1min":1.2909999999999999,"5min":1.4550000000000001,"15min":7.0659999999999998},"last":1.071},{"interface":"front","average":{"1min":1.0629999999999999,"5min":0.998,"15min":1.0549999999999999},"min":{"1min":0.76200000000000001,"5min":0.65600000000000003,"15min":0.33500000000000002},"max":{"1min":1.3660000000000001,"5min":1.546,"15min":5.992},"last":0.86099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:55:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.89000000000000001,"15min":2.585},"min":{"1min":0.67900000000000005,"5min":0.48299999999999998,"15min":0.28799999999999998},"max":{"1min":1.335,"5min":1.335,"15min":231.393},"last":1.097},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.92500000000000004,"15min":2.3929999999999998},"min":{"1min":0.63800000000000001,"5min":0.55600000000000005,"15min":0.29999999999999999},"max":{"1min":1.1850000000000001,"5min":1.637,"15min":231.17099999999999},"last":1.111}]},{"osd":3,"last update":"Mon Oct 9 06:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.98499999999999999,"15min":2.5249999999999999},"min":{"1min":0.80300000000000005,"5min":0.63600000000000001,"15min":0.34699999999999998},"max":{"1min":1.264,"5min":1.361,"15min":234.62200000000001},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.0389999999999999,"15min":2.6589999999999998},"min":{"1min":0.80000000000000004,"5min":0.55000000000000004,"15min":0.40400000000000003},"max":{"1min":1.256,"5min":1.51,"15min":234.673},"last":1.1479999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.94999999999999996,"15min":1.044},"min":{"1min":0.68600000000000005,"5min":0.57199999999999995,"15min":0.313},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":7.0330000000000004},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.97899999999999998,"15min":1.1679999999999999},"min":{"1min":0.73099999999999998,"5min":0.52900000000000003,"15min":0.27700000000000002},"max":{"1min":1.321,"5min":1.522,"15min":10.707000000000001},"last":0.94099999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0409999999999999,"15min":1.5289999999999999},"min":{"1min":0.83499999999999996,"5min":0.53000000000000003,"15min":0.38100000000000001},"max":{"1min":1.2709999999999999,"5min":1.8129999999999999,"15min":13.163},"last":0.88},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0680000000000001,"15min":1.337},"min":{"1min":0.78900000000000003,"5min":0.64000000000000001,"15min":0.373},"max":{"1min":1.3720000000000001,"5min":2.137,"15min":10.138999999999999},"last":0.82399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.093,"15min":1.6990000000000001},"min":{"1min":0.84499999999999997,"5min":0.629,"15min":0.312},"max":{"1min":1.3959999999999999,"5min":1.468,"15min":10.688000000000001},"last":1.004},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0569999999999999,"15min":1.1779999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.35599999999999998},"max":{"1min":1.335,"5min":1.476,"15min":8.9529999999999994},"last":0.92700000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823368,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597292,"kb_used_data":3492,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93147860,"statfs":{"total":95995035648,"available":95383408640,"internally_reserved":0,"allocated":3575808,"data_stored":2915346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.94199999999999995,"15min":2.762},"min":{"1min":0.59299999999999997,"5min":0.502,"15min":0.25800000000000001},"max":{"1min":1.2749999999999999,"5min":1.948,"15min":309.678},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.91300000000000003,"15min":3.028},"min":{"1min":0.59399999999999997,"5min":0.56100000000000005,"15min":0.29299999999999998},"max":{"1min":1.2789999999999999,"5min":1.3620000000000001,"15min":304.83300000000003},"last":0.94199999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:56:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.88200000000000001,"15min":0.93999999999999995},"min":{"1min":0.55000000000000004,"5min":0.52500000000000002,"15min":0.309},"max":{"1min":1.2230000000000001,"5min":1.28,"15min":6.7080000000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.90200000000000002,"15min":0.92200000000000004},"min":{"1min":0.628,"5min":0.48099999999999998,"15min":0.28299999999999997},"max":{"1min":1.2030000000000001,"5min":1.744,"15min":7.3330000000000002},"last":0.85899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94999999999999996,"15min":1.647},"min":{"1min":0.71599999999999997,"5min":0.497,"15min":0.34100000000000003},"max":{"1min":1.2729999999999999,"5min":2.6429999999999998,"15min":187.732},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97399999999999998,"15min":1.6970000000000001},"min":{"1min":0.72299999999999998,"5min":0.57799999999999996,"15min":0.33500000000000002},"max":{"1min":1.206,"5min":1.3819999999999999,"15min":188.286},"last":1.0669999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.99299999999999999,"15min":1.4199999999999999},"min":{"1min":0.65000000000000002,"5min":0.58799999999999997,"15min":0.34699999999999998},"max":{"1min":1.234,"5min":2.7290000000000001,"15min":48.509},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.036,"15min":1.371},"min":{"1min":0.90500000000000003,"5min":0.70799999999999996,"15min":0.377},"max":{"1min":1.2829999999999999,"5min":1.47,"15min":48.328000000000003},"last":0.82999999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:56:18 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.91300000000000003,"15min":1.155},"min":{"1min":0.71499999999999997,"5min":0.46300000000000002,"15min":0.33100000000000002},"max":{"1min":1.3859999999999999,"5min":2.0019999999999998,"15min":43.087000000000003},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0469999999999999,"15min":1.248},"min":{"1min":0.73999999999999999,"5min":0.496,"15min":0.34899999999999998},"max":{"1min":1.2929999999999999,"5min":3.3370000000000002,"15min":42.713000000000001},"last":1.0389999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.99099999999999999,"15min":1.036},"min":{"1min":0.72699999999999998,"5min":0.51600000000000001,"15min":0.36399999999999999},"max":{"1min":1.476,"5min":1.8220000000000001,"15min":7.3769999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.131,"15min":1.2749999999999999},"min":{"1min":0.77500000000000002,"5min":0.53800000000000003,"15min":0.371},"max":{"1min":1.349,"5min":2.8570000000000002,"15min":12.252000000000001},"last":1.1180000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1679999999999999,"15min":1.212},"min":{"1min":0.88200000000000001,"5min":0.56899999999999995,"15min":0.379},"max":{"1min":1.4670000000000001,"5min":2.7949999999999999,"15min":9.2200000000000006},"last":1.224},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.127,"15min":1.119},"min":{"1min":0.98099999999999998,"5min":0.57799999999999996,"15min":0.35199999999999998},"max":{"1min":1.349,"5min":1.5580000000000001,"15min":9.1950000000000003},"last":1.004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:56:34.021 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[80002]: audit 2023-10-09T06:56:33.043594+0000 mgr.y (mgr.25723) 1567 : audit 0 from='client.47027 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[80002]: audit 2023-10-09T06:56:33.623924+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[80002]: audit 2023-10-09T06:56:33.856416+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[80002]: audit 2023-10-09T06:56:34.233989+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[80002]: audit 2023-10-09T06:56:34.243402+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[77299]: audit 2023-10-09T06:56:33.043594+0000 mgr.y (mgr.25723) 1567 : audit 0 from='client.47027 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:34.890 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[77299]: audit 2023-10-09T06:56:33.623924+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:34.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[77299]: audit 2023-10-09T06:56:33.856416+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:56:34.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[77299]: audit 2023-10-09T06:56:34.233989+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:34.891 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:34 smithi125 bash[77299]: audit 2023-10-09T06:56:34.243402+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:34 smithi155 bash[59495]: audit 2023-10-09T06:56:33.043594+0000 mgr.y (mgr.25723) 1567 : audit 0 from='client.47027 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:34 smithi155 bash[59495]: audit 2023-10-09T06:56:33.623924+0000 mon.a (mon.0) 1933 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:56:34.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:34 smithi155 bash[59495]: audit 2023-10-09T06:56:33.856416+0000 mon.a (mon.0) 1934 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:56:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:34 smithi155 bash[59495]: audit 2023-10-09T06:56:34.233989+0000 mon.a (mon.0) 1935 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:34.894 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:34 smithi155 bash[59495]: audit 2023-10-09T06:56:34.243402+0000 mon.a (mon.0) 1936 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:35.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:35 smithi125 bash[77299]: cluster 2023-10-09T06:56:33.631853+0000 mgr.y (mgr.25723) 1568 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:35.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:35 smithi125 bash[80002]: cluster 2023-10-09T06:56:33.631853+0000 mgr.y (mgr.25723) 1568 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:35.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:35 smithi155 bash[59495]: cluster 2023-10-09T06:56:33.631853+0000 mgr.y (mgr.25723) 1568 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:37.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:37 smithi125 bash[77299]: cluster 2023-10-09T06:56:35.632785+0000 mgr.y (mgr.25723) 1569 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:37.887 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:37 smithi125 bash[80002]: cluster 2023-10-09T06:56:35.632785+0000 mgr.y (mgr.25723) 1569 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:37.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:37 smithi155 bash[59495]: cluster 2023-10-09T06:56:35.632785+0000 mgr.y (mgr.25723) 1569 : cluster 0 pgmap v1773: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:38.747 INFO:teuthology.orchestra.run.smithi125.stderr:Inferring config /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:56:39.886 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:39 smithi125 bash[77299]: cluster 2023-10-09T06:56:37.633638+0000 mgr.y (mgr.25723) 1570 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:39.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:39 smithi125 bash[80002]: cluster 2023-10-09T06:56:37.633638+0000 mgr.y (mgr.25723) 1570 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:39.893 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:39 smithi155 bash[59495]: cluster 2023-10-09T06:56:37.633638+0000 mgr.y (mgr.25723) 1570 : cluster 0 pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:41.112 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[77299]: cluster 2023-10-09T06:56:39.634595+0000 mgr.y (mgr.25723) 1571 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[77299]: audit 2023-10-09T06:56:39.856062+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[77299]: audit 2023-10-09T06:56:39.863339+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[77299]: audit 2023-10-09T06:56:40.534696+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[77299]: audit 2023-10-09T06:56:40.541749+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[80002]: cluster 2023-10-09T06:56:39.634595+0000 mgr.y (mgr.25723) 1571 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[80002]: audit 2023-10-09T06:56:39.856062+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[80002]: audit 2023-10-09T06:56:39.863339+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[80002]: audit 2023-10-09T06:56:40.534696+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.113 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:40 smithi125 bash[80002]: audit 2023-10-09T06:56:40.541749+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:40 smithi155 bash[59495]: cluster 2023-10-09T06:56:39.634595+0000 mgr.y (mgr.25723) 1571 : cluster 0 pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:40 smithi155 bash[59495]: audit 2023-10-09T06:56:39.856062+0000 mon.a (mon.0) 1937 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:40 smithi155 bash[59495]: audit 2023-10-09T06:56:39.863339+0000 mon.a (mon.0) 1938 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:40 smithi155 bash[59495]: audit 2023-10-09T06:56:40.534696+0000 mon.a (mon.0) 1939 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:40 smithi155 bash[59495]: audit 2023-10-09T06:56:40.541749+0000 mon.a (mon.0) 1940 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:41.386 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[73481]: ::ffff:172.21.15.155 - - [09/Oct/2023:06:56:41] "GET /metrics HTTP/1.1" 200 34179 "" "Prometheus/2.33.4" 2023-10-09T06:56:42.136 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[77299]: audit 2023-10-09T06:56:41.010346+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:56:42.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[77299]: audit 2023-10-09T06:56:41.012183+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:56:42.137 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[77299]: audit 2023-10-09T06:56:41.021701+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:42.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[80002]: audit 2023-10-09T06:56:41.010346+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:56:42.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[80002]: audit 2023-10-09T06:56:41.012183+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:56:42.137 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:41 smithi125 bash[80002]: audit 2023-10-09T06:56:41.021701+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:41 smithi155 bash[59495]: audit 2023-10-09T06:56:41.010346+0000 mon.a (mon.0) 1941 : audit 0 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:56:42.143 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:41 smithi155 bash[59495]: audit 2023-10-09T06:56:41.012183+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:56:42.144 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:41 smithi155 bash[59495]: audit 2023-10-09T06:56:41.021701+0000 mon.a (mon.0) 1943 : audit 1 from='mgr.25723 172.21.15.125:0/1872394503' entity='mgr.y' 2023-10-09T06:56:42.740 INFO:teuthology.orchestra.run.smithi125.stdout: 2023-10-09T06:56:42.742 INFO:teuthology.orchestra.run.smithi125.stderr:dumped all 2023-10-09T06:56:43.386 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:43 smithi125 bash[77299]: cluster 2023-10-09T06:56:41.635404+0000 mgr.y (mgr.25723) 1572 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:43.387 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:43 smithi125 bash[80002]: cluster 2023-10-09T06:56:41.635404+0000 mgr.y (mgr.25723) 1572 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:43.393 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:43 smithi155 bash[59495]: cluster 2023-10-09T06:56:41.635404+0000 mgr.y (mgr.25723) 1572 : cluster 0 pgmap v1776: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:56:43.763 INFO:teuthology.orchestra.run.smithi125.stdout:{"pg_ready":true,"pg_map":{"version":1776,"stamp":"2023-10-09T06:56:41.634850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12192,"num_read_kb":25881,"num_write":1950,"num_write_kb":27871,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":595,"num_bytes_recovered":20351220,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6367264,"kb_used_data":57632,"kb_used_omap":30,"kb_used_meta":6309537,"kb_avail":743593952,"statfs":{"total":767960285184,"available":761440206848,"internally_reserved":0,"allocated":59015168,"data_stored":50650473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31633,"internal_metadata":6460965999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005288"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":423,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455528+0000","last_change":"2023-10-09T06:52:41.261259+0000","last_active":"2023-10-09T06:54:47.455528+0000","last_peered":"2023-10-09T06:54:47.455528+0000","last_clean":"2023-10-09T06:54:47.455528+0000","last_became_active":"2023-10-09T06:52:41.260731+0000","last_became_peered":"2023-10-09T06:52:41.260731+0000","last_unstale":"2023-10-09T06:54:47.455528+0000","last_undegraded":"2023-10-09T06:54:47.455528+0000","last_fullsized":"2023-10-09T06:54:47.455528+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670457+0000","last_change":"2023-10-09T06:52:41.258305+0000","last_active":"2023-10-09T06:54:56.670457+0000","last_peered":"2023-10-09T06:54:56.670457+0000","last_clean":"2023-10-09T06:54:56.670457+0000","last_became_active":"2023-10-09T06:52:41.257911+0000","last_became_peered":"2023-10-09T06:52:41.257911+0000","last_unstale":"2023-10-09T06:54:56.670457+0000","last_undegraded":"2023-10-09T06:54:56.670457+0000","last_fullsized":"2023-10-09T06:54:56.670457+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:24.252303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":70,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.798666+0000","last_change":"2023-10-09T06:52:53.666252+0000","last_active":"2023-10-09T06:54:57.798666+0000","last_peered":"2023-10-09T06:54:57.798666+0000","last_clean":"2023-10-09T06:54:57.798666+0000","last_became_active":"2023-10-09T06:52:53.665868+0000","last_became_peered":"2023-10-09T06:52:53.665868+0000","last_unstale":"2023-10-09T06:54:57.798666+0000","last_undegraded":"2023-10-09T06:54:57.798666+0000","last_fullsized":"2023-10-09T06:54:57.798666+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:50.509786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123823+0000","last_change":"2023-10-09T06:52:29.653622+0000","last_active":"2023-10-09T06:54:57.123823+0000","last_peered":"2023-10-09T06:54:57.123823+0000","last_clean":"2023-10-09T06:54:57.123823+0000","last_became_active":"2023-10-09T06:52:29.652927+0000","last_became_peered":"2023-10-09T06:52:29.652927+0000","last_unstale":"2023-10-09T06:54:57.123823+0000","last_undegraded":"2023-10-09T06:54:57.123823+0000","last_fullsized":"2023-10-09T06:54:57.123823+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:34:01.072762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669510+0000","last_change":"2023-10-09T06:52:41.249371+0000","last_active":"2023-10-09T06:54:56.669510+0000","last_peered":"2023-10-09T06:54:56.669510+0000","last_clean":"2023-10-09T06:54:56.669510+0000","last_became_active":"2023-10-09T06:52:41.249008+0000","last_became_peered":"2023-10-09T06:52:41.249008+0000","last_unstale":"2023-10-09T06:54:56.669510+0000","last_undegraded":"2023-10-09T06:54:56.669510+0000","last_fullsized":"2023-10-09T06:54:56.669510+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:57.074206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666433+0000","last_change":"2023-10-09T06:51:41.234140+0000","last_active":"2023-10-09T06:54:46.666433+0000","last_peered":"2023-10-09T06:54:46.666433+0000","last_clean":"2023-10-09T06:54:46.666433+0000","last_became_active":"2023-10-09T06:51:41.233841+0000","last_became_peered":"2023-10-09T06:51:41.233841+0000","last_unstale":"2023-10-09T06:54:46.666433+0000","last_undegraded":"2023-10-09T06:54:46.666433+0000","last_fullsized":"2023-10-09T06:54:46.666433+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":284,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123972+0000","last_change":"2023-10-09T06:52:29.658396+0000","last_active":"2023-10-09T06:54:57.123972+0000","last_peered":"2023-10-09T06:54:57.123972+0000","last_clean":"2023-10-09T06:54:57.123972+0000","last_became_active":"2023-10-09T06:52:29.658046+0000","last_became_peered":"2023-10-09T06:52:29.658046+0000","last_unstale":"2023-10-09T06:54:57.123972+0000","last_undegraded":"2023-10-09T06:54:57.123972+0000","last_fullsized":"2023-10-09T06:54:57.123972+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:42.469186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669512+0000","last_change":"2023-10-09T06:52:41.255390+0000","last_active":"2023-10-09T06:54:56.669512+0000","last_peered":"2023-10-09T06:54:56.669512+0000","last_clean":"2023-10-09T06:54:56.669512+0000","last_became_active":"2023-10-09T06:52:41.254984+0000","last_became_peered":"2023-10-09T06:52:41.254984+0000","last_unstale":"2023-10-09T06:54:56.669512+0000","last_undegraded":"2023-10-09T06:54:56.669512+0000","last_fullsized":"2023-10-09T06:54:56.669512+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:14.258020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":57,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455161+0000","last_change":"2023-10-09T06:52:41.261238+0000","last_active":"2023-10-09T06:54:47.455161+0000","last_peered":"2023-10-09T06:54:47.455161+0000","last_clean":"2023-10-09T06:54:47.455161+0000","last_became_active":"2023-10-09T06:52:41.260852+0000","last_became_peered":"2023-10-09T06:52:41.260852+0000","last_unstale":"2023-10-09T06:54:47.455161+0000","last_undegraded":"2023-10-09T06:54:47.455161+0000","last_fullsized":"2023-10-09T06:54:47.455161+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669737+0000","last_change":"2023-10-09T06:51:08.220112+0000","last_active":"2023-10-09T06:54:56.669737+0000","last_peered":"2023-10-09T06:54:56.669737+0000","last_clean":"2023-10-09T06:54:56.669737+0000","last_became_active":"2023-10-09T06:51:08.219059+0000","last_became_peered":"2023-10-09T06:51:08.219059+0000","last_unstale":"2023-10-09T06:54:56.669737+0000","last_undegraded":"2023-10-09T06:54:56.669737+0000","last_fullsized":"2023-10-09T06:54:56.669737+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:12.671358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":333,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181209+0000","last_change":"2023-10-09T06:51:08.360804+0000","last_active":"2023-10-09T06:54:47.181209+0000","last_peered":"2023-10-09T06:54:47.181209+0000","last_clean":"2023-10-09T06:54:47.181209+0000","last_became_active":"2023-10-09T06:51:08.221588+0000","last_became_peered":"2023-10-09T06:51:08.221588+0000","last_unstale":"2023-10-09T06:54:47.181209+0000","last_undegraded":"2023-10-09T06:54:47.181209+0000","last_fullsized":"2023-10-09T06:54:47.181209+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":186,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799561+0000","last_change":"2023-10-09T06:52:54.127043+0000","last_active":"2023-10-09T06:54:57.799561+0000","last_peered":"2023-10-09T06:54:57.799561+0000","last_clean":"2023-10-09T06:54:57.799561+0000","last_became_active":"2023-10-09T06:52:54.126228+0000","last_became_peered":"2023-10-09T06:52:54.126228+0000","last_unstale":"2023-10-09T06:54:57.799561+0000","last_undegraded":"2023-10-09T06:54:57.799561+0000","last_fullsized":"2023-10-09T06:54:57.799561+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:30.427085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"84'22","reported_seq":492,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455880+0000","last_change":"2023-10-09T06:42:32.588180+0000","last_active":"2023-10-09T06:54:47.455880+0000","last_peered":"2023-10-09T06:54:47.455880+0000","last_clean":"2023-10-09T06:54:47.455880+0000","last_became_active":"2023-10-09T06:42:32.482690+0000","last_became_peered":"2023-10-09T06:42:32.482690+0000","last_unstale":"2023-10-09T06:54:47.455880+0000","last_undegraded":"2023-10-09T06:54:47.455880+0000","last_fullsized":"2023-10-09T06:54:47.455880+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":451,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455837+0000","last_change":"2023-10-09T06:53:22.252174+0000","last_active":"2023-10-09T06:54:47.455837+0000","last_peered":"2023-10-09T06:54:47.455837+0000","last_clean":"2023-10-09T06:54:47.455837+0000","last_became_active":"2023-10-09T06:53:22.251093+0000","last_became_peered":"2023-10-09T06:53:22.251093+0000","last_unstale":"2023-10-09T06:54:47.455837+0000","last_undegraded":"2023-10-09T06:54:47.455837+0000","last_fullsized":"2023-10-09T06:54:47.455837+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":333,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180598+0000","last_change":"2023-10-09T06:51:53.829980+0000","last_active":"2023-10-09T06:54:47.180598+0000","last_peered":"2023-10-09T06:54:47.180598+0000","last_clean":"2023-10-09T06:54:47.180598+0000","last_became_active":"2023-10-09T06:51:53.655619+0000","last_became_peered":"2023-10-09T06:51:53.655619+0000","last_unstale":"2023-10-09T06:54:47.180598+0000","last_undegraded":"2023-10-09T06:54:47.180598+0000","last_fullsized":"2023-10-09T06:54:47.180598+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":44,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455204+0000","last_change":"2023-10-09T06:52:41.266974+0000","last_active":"2023-10-09T06:54:47.455204+0000","last_peered":"2023-10-09T06:54:47.455204+0000","last_clean":"2023-10-09T06:54:47.455204+0000","last_became_active":"2023-10-09T06:52:41.266599+0000","last_became_peered":"2023-10-09T06:52:41.266599+0000","last_unstale":"2023-10-09T06:54:47.455204+0000","last_undegraded":"2023-10-09T06:54:47.455204+0000","last_fullsized":"2023-10-09T06:54:47.455204+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":248,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:53.059335+0000","last_change":"2023-10-09T06:52:29.651724+0000","last_active":"2023-10-09T06:54:53.059335+0000","last_peered":"2023-10-09T06:54:53.059335+0000","last_clean":"2023-10-09T06:54:53.059335+0000","last_became_active":"2023-10-09T06:52:29.651303+0000","last_became_peered":"2023-10-09T06:52:29.651303+0000","last_unstale":"2023-10-09T06:54:53.059335+0000","last_undegraded":"2023-10-09T06:54:53.059335+0000","last_fullsized":"2023-10-09T06:54:53.059335+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:53.112045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455477+0000","last_change":"2023-10-09T06:51:41.236068+0000","last_active":"2023-10-09T06:54:47.455477+0000","last_peered":"2023-10-09T06:54:47.455477+0000","last_clean":"2023-10-09T06:54:47.455477+0000","last_became_active":"2023-10-09T06:51:41.235740+0000","last_became_peered":"2023-10-09T06:51:41.235740+0000","last_unstale":"2023-10-09T06:54:47.455477+0000","last_undegraded":"2023-10-09T06:54:47.455477+0000","last_fullsized":"2023-10-09T06:54:47.455477+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":351,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180589+0000","last_change":"2023-10-09T06:45:05.141190+0000","last_active":"2023-10-09T06:54:47.180589+0000","last_peered":"2023-10-09T06:54:47.180589+0000","last_clean":"2023-10-09T06:54:47.180589+0000","last_became_active":"2023-10-09T06:45:05.140792+0000","last_became_peered":"2023-10-09T06:45:05.140792+0000","last_unstale":"2023-10-09T06:54:47.180589+0000","last_undegraded":"2023-10-09T06:54:47.180589+0000","last_fullsized":"2023-10-09T06:54:47.180589+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":248,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123929+0000","last_change":"2023-10-09T06:52:29.653842+0000","last_active":"2023-10-09T06:54:57.123929+0000","last_peered":"2023-10-09T06:54:57.123929+0000","last_clean":"2023-10-09T06:54:57.123929+0000","last_became_active":"2023-10-09T06:52:29.652933+0000","last_became_peered":"2023-10-09T06:52:29.652933+0000","last_unstale":"2023-10-09T06:54:57.123929+0000","last_undegraded":"2023-10-09T06:54:57.123929+0000","last_fullsized":"2023-10-09T06:54:57.123929+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:26.998073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":239,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666109+0000","last_change":"2023-10-09T06:51:41.413465+0000","last_active":"2023-10-09T06:54:46.666109+0000","last_peered":"2023-10-09T06:54:46.666109+0000","last_clean":"2023-10-09T06:54:46.666109+0000","last_became_active":"2023-10-09T06:51:41.238496+0000","last_became_peered":"2023-10-09T06:51:41.238496+0000","last_unstale":"2023-10-09T06:54:46.666109+0000","last_undegraded":"2023-10-09T06:54:46.666109+0000","last_fullsized":"2023-10-09T06:54:46.666109+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670753+0000","last_change":"2023-10-09T06:53:22.251307+0000","last_active":"2023-10-09T06:54:56.670753+0000","last_peered":"2023-10-09T06:54:56.670753+0000","last_clean":"2023-10-09T06:54:56.670753+0000","last_became_active":"2023-10-09T06:53:22.250530+0000","last_became_peered":"2023-10-09T06:53:22.250530+0000","last_unstale":"2023-10-09T06:54:56.670753+0000","last_undegraded":"2023-10-09T06:54:56.670753+0000","last_fullsized":"2023-10-09T06:54:56.670753+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:32.096800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":301,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181329+0000","last_change":"2023-10-09T06:19:00.078836+0000","last_active":"2023-10-09T06:54:47.181329+0000","last_peered":"2023-10-09T06:54:47.181329+0000","last_clean":"2023-10-09T06:54:47.181329+0000","last_became_active":"2023-10-09T06:18:59.809771+0000","last_became_peered":"2023-10-09T06:18:59.809771+0000","last_unstale":"2023-10-09T06:54:47.181329+0000","last_undegraded":"2023-10-09T06:54:47.181329+0000","last_fullsized":"2023-10-09T06:54:47.181329+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.454943+0000","last_change":"2023-10-09T06:51:41.231467+0000","last_active":"2023-10-09T06:54:47.454943+0000","last_peered":"2023-10-09T06:54:47.454943+0000","last_clean":"2023-10-09T06:54:47.454943+0000","last_became_active":"2023-10-09T06:51:41.231125+0000","last_became_peered":"2023-10-09T06:51:41.231125+0000","last_unstale":"2023-10-09T06:54:47.454943+0000","last_undegraded":"2023-10-09T06:54:47.454943+0000","last_fullsized":"2023-10-09T06:54:47.454943+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":353,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181152+0000","last_change":"2023-10-09T06:51:26.276815+0000","last_active":"2023-10-09T06:54:47.181152+0000","last_peered":"2023-10-09T06:54:47.181152+0000","last_clean":"2023-10-09T06:54:47.181152+0000","last_became_active":"2023-10-09T06:51:26.229980+0000","last_became_peered":"2023-10-09T06:51:26.229980+0000","last_unstale":"2023-10-09T06:54:47.181152+0000","last_undegraded":"2023-10-09T06:54:47.181152+0000","last_fullsized":"2023-10-09T06:54:47.181152+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":430,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455358+0000","last_change":"2023-10-09T06:52:41.261783+0000","last_active":"2023-10-09T06:54:47.455358+0000","last_peered":"2023-10-09T06:54:47.455358+0000","last_clean":"2023-10-09T06:54:47.455358+0000","last_became_active":"2023-10-09T06:52:41.261385+0000","last_became_peered":"2023-10-09T06:52:41.261385+0000","last_unstale":"2023-10-09T06:54:47.455358+0000","last_undegraded":"2023-10-09T06:54:47.455358+0000","last_fullsized":"2023-10-09T06:54:47.455358+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":405,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180971+0000","last_change":"2023-10-09T06:51:41.458272+0000","last_active":"2023-10-09T06:54:47.180971+0000","last_peered":"2023-10-09T06:54:47.180971+0000","last_clean":"2023-10-09T06:54:47.180971+0000","last_became_active":"2023-10-09T06:51:41.236211+0000","last_became_peered":"2023-10-09T06:51:41.236211+0000","last_unstale":"2023-10-09T06:54:47.180971+0000","last_undegraded":"2023-10-09T06:54:47.180971+0000","last_fullsized":"2023-10-09T06:54:47.180971+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666040+0000","last_change":"2023-10-09T06:52:21.262418+0000","last_active":"2023-10-09T06:54:46.666040+0000","last_peered":"2023-10-09T06:54:46.666040+0000","last_clean":"2023-10-09T06:54:46.666040+0000","last_became_active":"2023-10-09T06:52:21.261834+0000","last_became_peered":"2023-10-09T06:52:21.261834+0000","last_unstale":"2023-10-09T06:54:46.666040+0000","last_undegraded":"2023-10-09T06:54:46.666040+0000","last_fullsized":"2023-10-09T06:54:46.666040+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455476+0000","last_change":"2023-10-09T06:51:41.233437+0000","last_active":"2023-10-09T06:54:47.455476+0000","last_peered":"2023-10-09T06:54:47.455476+0000","last_clean":"2023-10-09T06:54:47.455476+0000","last_became_active":"2023-10-09T06:51:41.231493+0000","last_became_peered":"2023-10-09T06:51:41.231493+0000","last_unstale":"2023-10-09T06:54:47.455476+0000","last_undegraded":"2023-10-09T06:54:47.455476+0000","last_fullsized":"2023-10-09T06:54:47.455476+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":378,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:48.058470+0000","last_change":"2023-10-09T06:52:29.663419+0000","last_active":"2023-10-09T06:54:48.058470+0000","last_peered":"2023-10-09T06:54:48.058470+0000","last_clean":"2023-10-09T06:54:48.058470+0000","last_became_active":"2023-10-09T06:52:29.663061+0000","last_became_peered":"2023-10-09T06:52:29.663061+0000","last_unstale":"2023-10-09T06:54:48.058470+0000","last_undegraded":"2023-10-09T06:54:48.058470+0000","last_fullsized":"2023-10-09T06:54:48.058470+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:48:07.866649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669930+0000","last_change":"2023-10-09T06:51:08.233038+0000","last_active":"2023-10-09T06:54:56.669930+0000","last_peered":"2023-10-09T06:54:56.669930+0000","last_clean":"2023-10-09T06:54:56.669930+0000","last_became_active":"2023-10-09T06:51:08.221188+0000","last_became_peered":"2023-10-09T06:51:08.221188+0000","last_unstale":"2023-10-09T06:54:56.669930+0000","last_undegraded":"2023-10-09T06:54:56.669930+0000","last_fullsized":"2023-10-09T06:54:56.669930+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:01.269604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":424,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455901+0000","last_change":"2023-10-09T06:51:08.227179+0000","last_active":"2023-10-09T06:54:47.455901+0000","last_peered":"2023-10-09T06:54:47.455901+0000","last_clean":"2023-10-09T06:54:47.455901+0000","last_became_active":"2023-10-09T06:51:08.226782+0000","last_became_peered":"2023-10-09T06:51:08.226782+0000","last_unstale":"2023-10-09T06:54:47.455901+0000","last_undegraded":"2023-10-09T06:54:47.455901+0000","last_fullsized":"2023-10-09T06:54:47.455901+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666268+0000","last_change":"2023-10-09T06:54:38.034620+0000","last_active":"2023-10-09T06:54:46.666268+0000","last_peered":"2023-10-09T06:54:46.666268+0000","last_clean":"2023-10-09T06:54:46.666268+0000","last_became_active":"2023-10-09T06:54:38.034101+0000","last_became_peered":"2023-10-09T06:54:38.034101+0000","last_unstale":"2023-10-09T06:54:46.666268+0000","last_undegraded":"2023-10-09T06:54:46.666268+0000","last_fullsized":"2023-10-09T06:54:46.666268+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455969+0000","last_change":"2023-10-09T06:51:53.658954+0000","last_active":"2023-10-09T06:54:47.455969+0000","last_peered":"2023-10-09T06:54:47.455969+0000","last_clean":"2023-10-09T06:54:47.455969+0000","last_became_active":"2023-10-09T06:51:53.658393+0000","last_became_peered":"2023-10-09T06:51:53.658393+0000","last_unstale":"2023-10-09T06:54:47.455969+0000","last_undegraded":"2023-10-09T06:54:47.455969+0000","last_fullsized":"2023-10-09T06:54:47.455969+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455157+0000","last_change":"2023-10-09T06:51:53.656986+0000","last_active":"2023-10-09T06:54:47.455157+0000","last_peered":"2023-10-09T06:54:47.455157+0000","last_clean":"2023-10-09T06:54:47.455157+0000","last_became_active":"2023-10-09T06:51:53.656411+0000","last_became_peered":"2023-10-09T06:51:53.656411+0000","last_unstale":"2023-10-09T06:54:47.455157+0000","last_undegraded":"2023-10-09T06:54:47.455157+0000","last_fullsized":"2023-10-09T06:54:47.455157+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":442,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.456560+0000","last_change":"2023-10-09T06:52:41.266484+0000","last_active":"2023-10-09T06:54:47.456560+0000","last_peered":"2023-10-09T06:54:47.456560+0000","last_clean":"2023-10-09T06:54:47.456560+0000","last_became_active":"2023-10-09T06:52:41.261618+0000","last_became_peered":"2023-10-09T06:52:41.261618+0000","last_unstale":"2023-10-09T06:54:47.456560+0000","last_undegraded":"2023-10-09T06:54:47.456560+0000","last_fullsized":"2023-10-09T06:54:47.456560+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669690+0000","last_change":"2023-10-09T06:52:41.249404+0000","last_active":"2023-10-09T06:54:56.669690+0000","last_peered":"2023-10-09T06:54:56.669690+0000","last_clean":"2023-10-09T06:54:56.669690+0000","last_became_active":"2023-10-09T06:52:41.249078+0000","last_became_peered":"2023-10-09T06:52:41.249078+0000","last_unstale":"2023-10-09T06:54:56.669690+0000","last_undegraded":"2023-10-09T06:54:56.669690+0000","last_fullsized":"2023-10-09T06:54:56.669690+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:23.696492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799309+0000","last_change":"2023-10-09T06:52:53.666300+0000","last_active":"2023-10-09T06:54:57.799309+0000","last_peered":"2023-10-09T06:54:57.799309+0000","last_clean":"2023-10-09T06:54:57.799309+0000","last_became_active":"2023-10-09T06:52:53.665861+0000","last_became_peered":"2023-10-09T06:52:53.665861+0000","last_unstale":"2023-10-09T06:54:57.799309+0000","last_undegraded":"2023-10-09T06:54:57.799309+0000","last_fullsized":"2023-10-09T06:54:57.799309+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:02.601488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"350'344","reported_seq":1884,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799503+0000","last_change":"2023-10-09T06:52:53.666148+0000","last_active":"2023-10-09T06:54:57.799503+0000","last_peered":"2023-10-09T06:54:57.799503+0000","last_clean":"2023-10-09T06:54:57.799503+0000","last_became_active":"2023-10-09T06:52:53.665155+0000","last_became_peered":"2023-10-09T06:52:53.665155+0000","last_unstale":"2023-10-09T06:54:57.799503+0000","last_undegraded":"2023-10-09T06:54:57.799503+0000","last_fullsized":"2023-10-09T06:54:57.799503+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":344,"log_dups_size":0,"ondisk_log_size":344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:44.013378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2584,"num_read_kb":4613,"num_write":301,"num_write_kb":2356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":57,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455398+0000","last_change":"2023-10-09T06:51:53.654357+0000","last_active":"2023-10-09T06:54:47.455398+0000","last_peered":"2023-10-09T06:54:47.455398+0000","last_clean":"2023-10-09T06:54:47.455398+0000","last_became_active":"2023-10-09T06:51:53.653956+0000","last_became_peered":"2023-10-09T06:51:53.653956+0000","last_unstale":"2023-10-09T06:54:47.455398+0000","last_undegraded":"2023-10-09T06:54:47.455398+0000","last_fullsized":"2023-10-09T06:54:47.455398+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.798748+0000","last_change":"2023-10-09T06:52:54.125962+0000","last_active":"2023-10-09T06:54:57.798748+0000","last_peered":"2023-10-09T06:54:57.798748+0000","last_clean":"2023-10-09T06:54:57.798748+0000","last_became_active":"2023-10-09T06:52:54.125101+0000","last_became_peered":"2023-10-09T06:52:54.125101+0000","last_unstale":"2023-10-09T06:54:57.798748+0000","last_undegraded":"2023-10-09T06:54:57.798748+0000","last_fullsized":"2023-10-09T06:54:57.798748+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:32.735554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"350'187","reported_seq":697,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455884+0000","last_change":"2023-10-09T06:52:45.244732+0000","last_active":"2023-10-09T06:54:47.455884+0000","last_peered":"2023-10-09T06:54:47.455884+0000","last_clean":"2023-10-09T06:54:47.455884+0000","last_became_active":"2023-10-09T06:52:45.243723+0000","last_became_peered":"2023-10-09T06:52:45.243723+0000","last_unstale":"2023-10-09T06:54:47.455884+0000","last_undegraded":"2023-10-09T06:54:47.455884+0000","last_fullsized":"2023-10-09T06:54:47.455884+0000","mapping_epoch":387,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":187,"log_dups_size":0,"ondisk_log_size":187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1616,"num_read_kb":3123,"num_write":176,"num_write_kb":2470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":416,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455744+0000","last_change":"2023-10-09T06:42:32.474987+0000","last_active":"2023-10-09T06:54:47.455744+0000","last_peered":"2023-10-09T06:54:47.455744+0000","last_clean":"2023-10-09T06:54:47.455744+0000","last_became_active":"2023-10-09T06:42:32.474660+0000","last_became_peered":"2023-10-09T06:42:32.474660+0000","last_unstale":"2023-10-09T06:54:47.455744+0000","last_undegraded":"2023-10-09T06:54:47.455744+0000","last_fullsized":"2023-10-09T06:54:47.455744+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":60,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799423+0000","last_change":"2023-10-09T06:52:54.126780+0000","last_active":"2023-10-09T06:54:57.799423+0000","last_peered":"2023-10-09T06:54:57.799423+0000","last_clean":"2023-10-09T06:54:57.799423+0000","last_became_active":"2023-10-09T06:52:54.125995+0000","last_became_peered":"2023-10-09T06:52:54.125995+0000","last_unstale":"2023-10-09T06:54:57.799423+0000","last_undegraded":"2023-10-09T06:54:57.799423+0000","last_fullsized":"2023-10-09T06:54:57.799423+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:19.851665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"350'186","reported_seq":822,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.475930+0000","last_change":"2023-10-09T06:51:56.673791+0000","last_active":"2023-10-09T06:54:57.475930+0000","last_peered":"2023-10-09T06:54:57.475930+0000","last_clean":"2023-10-09T06:54:57.475930+0000","last_became_active":"2023-10-09T06:51:56.672920+0000","last_became_peered":"2023-10-09T06:51:56.672920+0000","last_unstale":"2023-10-09T06:54:57.475930+0000","last_undegraded":"2023-10-09T06:54:57.475930+0000","last_fullsized":"2023-10-09T06:54:57.475930+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":186,"log_dups_size":0,"ondisk_log_size":186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:33:40.096965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":933,"num_read_kb":1099,"num_write":169,"num_write_kb":4412,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":51,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.510966+0000","last_change":"2023-10-09T06:52:54.126091+0000","last_active":"2023-10-09T06:54:56.510966+0000","last_peered":"2023-10-09T06:54:56.510966+0000","last_clean":"2023-10-09T06:54:56.510966+0000","last_became_active":"2023-10-09T06:52:54.125482+0000","last_became_peered":"2023-10-09T06:52:54.125482+0000","last_unstale":"2023-10-09T06:54:56.510966+0000","last_undegraded":"2023-10-09T06:54:56.510966+0000","last_fullsized":"2023-10-09T06:54:56.510966+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:40.982672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799213+0000","last_change":"2023-10-09T06:52:54.125788+0000","last_active":"2023-10-09T06:54:57.799213+0000","last_peered":"2023-10-09T06:54:57.799213+0000","last_clean":"2023-10-09T06:54:57.799213+0000","last_became_active":"2023-10-09T06:52:54.124991+0000","last_became_peered":"2023-10-09T06:52:54.124991+0000","last_unstale":"2023-10-09T06:54:57.799213+0000","last_undegraded":"2023-10-09T06:54:57.799213+0000","last_fullsized":"2023-10-09T06:54:57.799213+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:56:11.624260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"350'307","reported_seq":1097,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455252+0000","last_change":"2023-10-09T06:51:58.697612+0000","last_active":"2023-10-09T06:54:47.455252+0000","last_peered":"2023-10-09T06:54:47.455252+0000","last_clean":"2023-10-09T06:54:47.455252+0000","last_became_active":"2023-10-09T06:51:58.696659+0000","last_became_peered":"2023-10-09T06:51:58.696659+0000","last_unstale":"2023-10-09T06:54:47.455252+0000","last_undegraded":"2023-10-09T06:54:47.455252+0000","last_fullsized":"2023-10-09T06:54:47.455252+0000","mapping_epoch":377,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":307,"log_dups_size":0,"ondisk_log_size":307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":752,"num_read_kb":4990,"num_write":302,"num_write_kb":3916,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"84'139","reported_seq":700,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180621+0000","last_change":"2023-10-09T06:42:34.466448+0000","last_active":"2023-10-09T06:54:47.180621+0000","last_peered":"2023-10-09T06:54:47.180621+0000","last_clean":"2023-10-09T06:54:47.180621+0000","last_became_active":"2023-10-09T06:42:34.466006+0000","last_became_peered":"2023-10-09T06:42:34.466006+0000","last_unstale":"2023-10-09T06:54:47.180621+0000","last_undegraded":"2023-10-09T06:54:47.180621+0000","last_fullsized":"2023-10-09T06:54:47.180621+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:55:15.725276+0000","last_clean_scrub_stamp":"2023-10-09T05:55:15.725276+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":74,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669912+0000","last_change":"2023-10-09T06:51:41.233039+0000","last_active":"2023-10-09T06:54:56.669912+0000","last_peered":"2023-10-09T06:54:56.669912+0000","last_clean":"2023-10-09T06:54:56.669912+0000","last_became_active":"2023-10-09T06:51:41.232622+0000","last_became_peered":"2023-10-09T06:51:41.232622+0000","last_unstale":"2023-10-09T06:54:56.669912+0000","last_undegraded":"2023-10-09T06:54:56.669912+0000","last_fullsized":"2023-10-09T06:54:56.669912+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:46:40.553458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666349+0000","last_change":"2023-10-09T06:54:38.035139+0000","last_active":"2023-10-09T06:54:46.666349+0000","last_peered":"2023-10-09T06:54:46.666349+0000","last_clean":"2023-10-09T06:54:46.666349+0000","last_became_active":"2023-10-09T06:54:38.034786+0000","last_became_peered":"2023-10-09T06:54:38.034786+0000","last_unstale":"2023-10-09T06:54:46.666349+0000","last_undegraded":"2023-10-09T06:54:46.666349+0000","last_fullsized":"2023-10-09T06:54:46.666349+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"350'198","reported_seq":944,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.181106+0000","last_change":"2023-10-09T06:52:46.247372+0000","last_active":"2023-10-09T06:54:47.181106+0000","last_peered":"2023-10-09T06:54:47.181106+0000","last_clean":"2023-10-09T06:54:47.181106+0000","last_became_active":"2023-10-09T06:52:46.246564+0000","last_became_peered":"2023-10-09T06:52:46.246564+0000","last_unstale":"2023-10-09T06:54:47.181106+0000","last_undegraded":"2023-10-09T06:54:47.181106+0000","last_fullsized":"2023-10-09T06:54:47.181106+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":198,"log_dups_size":0,"ondisk_log_size":198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1395,"num_read_kb":2773,"num_write":167,"num_write_kb":1385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":277,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450224+0000","last_change":"2023-10-09T06:52:29.659138+0000","last_active":"2023-10-09T06:54:47.450224+0000","last_peered":"2023-10-09T06:54:47.450224+0000","last_clean":"2023-10-09T06:54:47.450224+0000","last_became_active":"2023-10-09T06:52:29.658791+0000","last_became_peered":"2023-10-09T06:52:29.658791+0000","last_unstale":"2023-10-09T06:54:47.450224+0000","last_undegraded":"2023-10-09T06:54:47.450224+0000","last_fullsized":"2023-10-09T06:54:47.450224+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:55:52.081375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":411,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450302+0000","last_change":"2023-10-09T06:52:41.248968+0000","last_active":"2023-10-09T06:54:47.450302+0000","last_peered":"2023-10-09T06:54:47.450302+0000","last_clean":"2023-10-09T06:54:47.450302+0000","last_became_active":"2023-10-09T06:52:41.248572+0000","last_became_peered":"2023-10-09T06:52:41.248572+0000","last_unstale":"2023-10-09T06:54:47.450302+0000","last_undegraded":"2023-10-09T06:54:47.450302+0000","last_fullsized":"2023-10-09T06:54:47.450302+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:38.052232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"350'194","reported_seq":748,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.180869+0000","last_change":"2023-10-09T06:50:42.202899+0000","last_active":"2023-10-09T06:54:47.180869+0000","last_peered":"2023-10-09T06:54:47.180869+0000","last_clean":"2023-10-09T06:54:47.180869+0000","last_became_active":"2023-10-09T06:18:59.810879+0000","last_became_peered":"2023-10-09T06:18:59.810879+0000","last_unstale":"2023-10-09T06:54:47.180869+0000","last_undegraded":"2023-10-09T06:54:47.180869+0000","last_fullsized":"2023-10-09T06:54:47.180869+0000","mapping_epoch":117,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1332,"num_read_kb":3158,"num_write":169,"num_write_kb":2120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":51,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799598+0000","last_change":"2023-10-09T06:52:54.126837+0000","last_active":"2023-10-09T06:54:57.799598+0000","last_peered":"2023-10-09T06:54:57.799598+0000","last_clean":"2023-10-09T06:54:57.799598+0000","last_became_active":"2023-10-09T06:52:54.126361+0000","last_became_peered":"2023-10-09T06:52:54.126361+0000","last_unstale":"2023-10-09T06:54:57.799598+0000","last_undegraded":"2023-10-09T06:54:57.799598+0000","last_fullsized":"2023-10-09T06:54:57.799598+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:08:44.013804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.452498+0000","last_change":"2023-10-09T06:52:29.660430+0000","last_active":"2023-10-09T06:54:47.452498+0000","last_peered":"2023-10-09T06:54:47.452498+0000","last_clean":"2023-10-09T06:54:47.452498+0000","last_became_active":"2023-10-09T06:52:29.660175+0000","last_became_peered":"2023-10-09T06:52:29.660175+0000","last_unstale":"2023-10-09T06:54:47.452498+0000","last_undegraded":"2023-10-09T06:54:47.452498+0000","last_fullsized":"2023-10-09T06:54:47.452498+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:27.353492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670469+0000","last_change":"2023-10-09T06:51:08.224948+0000","last_active":"2023-10-09T06:54:56.670469+0000","last_peered":"2023-10-09T06:54:56.670469+0000","last_clean":"2023-10-09T06:54:56.670469+0000","last_became_active":"2023-10-09T06:51:08.212213+0000","last_became_peered":"2023-10-09T06:51:08.212213+0000","last_unstale":"2023-10-09T06:54:56.670469+0000","last_undegraded":"2023-10-09T06:54:56.670469+0000","last_fullsized":"2023-10-09T06:54:56.670469+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:55.420006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"350'241","reported_seq":1306,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.799369+0000","last_change":"2023-10-09T06:52:53.667462+0000","last_active":"2023-10-09T06:54:57.799369+0000","last_peered":"2023-10-09T06:54:57.799369+0000","last_clean":"2023-10-09T06:54:57.799369+0000","last_became_active":"2023-10-09T06:52:53.666474+0000","last_became_peered":"2023-10-09T06:52:53.666474+0000","last_unstale":"2023-10-09T06:54:57.799369+0000","last_undegraded":"2023-10-09T06:54:57.799369+0000","last_fullsized":"2023-10-09T06:54:57.799369+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":241,"log_dups_size":0,"ondisk_log_size":241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:53.594480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1357,"num_read_kb":2514,"num_write":167,"num_write_kb":5394,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.475823+0000","last_change":"2023-10-09T06:52:21.261388+0000","last_active":"2023-10-09T06:54:57.475823+0000","last_peered":"2023-10-09T06:54:57.475823+0000","last_clean":"2023-10-09T06:54:57.475823+0000","last_became_active":"2023-10-09T06:52:21.260902+0000","last_became_peered":"2023-10-09T06:52:21.260902+0000","last_unstale":"2023-10-09T06:54:57.475823+0000","last_undegraded":"2023-10-09T06:54:57.475823+0000","last_fullsized":"2023-10-09T06:54:57.475823+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:09.200069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"350'293","reported_seq":1503,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:46.666529+0000","last_change":"2023-10-09T06:51:56.678019+0000","last_active":"2023-10-09T06:54:46.666529+0000","last_peered":"2023-10-09T06:54:46.666529+0000","last_clean":"2023-10-09T06:54:46.666529+0000","last_became_active":"2023-10-09T06:51:56.677244+0000","last_became_peered":"2023-10-09T06:51:56.677244+0000","last_unstale":"2023-10-09T06:54:46.666529+0000","last_undegraded":"2023-10-09T06:54:46.666529+0000","last_fullsized":"2023-10-09T06:54:46.666529+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:59:51.436952+0000","last_clean_scrub_stamp":"2023-10-09T05:59:51.436952+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1929,"num_read_kb":3111,"num_write":239,"num_write_kb":3373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":306,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450522+0000","last_change":"2023-10-09T06:52:41.278834+0000","last_active":"2023-10-09T06:54:47.450522+0000","last_peered":"2023-10-09T06:54:47.450522+0000","last_clean":"2023-10-09T06:54:47.450522+0000","last_became_active":"2023-10-09T06:52:41.259514+0000","last_became_peered":"2023-10-09T06:52:41.259514+0000","last_unstale":"2023-10-09T06:54:47.450522+0000","last_undegraded":"2023-10-09T06:54:47.450522+0000","last_fullsized":"2023-10-09T06:54:47.450522+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:32.900490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":409,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455255+0000","last_change":"2023-10-09T06:51:41.229292+0000","last_active":"2023-10-09T06:54:47.455255+0000","last_peered":"2023-10-09T06:54:47.455255+0000","last_clean":"2023-10-09T06:54:47.455255+0000","last_became_active":"2023-10-09T06:51:41.228949+0000","last_became_peered":"2023-10-09T06:51:41.228949+0000","last_unstale":"2023-10-09T06:54:47.455255+0000","last_undegraded":"2023-10-09T06:54:47.455255+0000","last_fullsized":"2023-10-09T06:54:47.455255+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:57.123779+0000","last_change":"2023-10-09T06:52:29.660475+0000","last_active":"2023-10-09T06:54:57.123779+0000","last_peered":"2023-10-09T06:54:57.123779+0000","last_clean":"2023-10-09T06:54:57.123779+0000","last_became_active":"2023-10-09T06:52:29.660091+0000","last_became_peered":"2023-10-09T06:52:29.660091+0000","last_unstale":"2023-10-09T06:54:57.123779+0000","last_undegraded":"2023-10-09T06:54:57.123779+0000","last_fullsized":"2023-10-09T06:54:57.123779+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:29.940623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":45,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450787+0000","last_change":"2023-10-09T06:51:53.650329+0000","last_active":"2023-10-09T06:54:47.450787+0000","last_peered":"2023-10-09T06:54:47.450787+0000","last_clean":"2023-10-09T06:54:47.450787+0000","last_became_active":"2023-10-09T06:51:53.649959+0000","last_became_peered":"2023-10-09T06:51:53.649959+0000","last_unstale":"2023-10-09T06:54:47.450787+0000","last_undegraded":"2023-10-09T06:54:47.450787+0000","last_fullsized":"2023-10-09T06:54:47.450787+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:08.498803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":65,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455297+0000","last_change":"2023-10-09T06:52:41.327440+0000","last_active":"2023-10-09T06:54:47.455297+0000","last_peered":"2023-10-09T06:54:47.455297+0000","last_clean":"2023-10-09T06:54:47.455297+0000","last_became_active":"2023-10-09T06:52:41.266476+0000","last_became_peered":"2023-10-09T06:52:41.266476+0000","last_unstale":"2023-10-09T06:54:47.455297+0000","last_undegraded":"2023-10-09T06:54:47.455297+0000","last_fullsized":"2023-10-09T06:54:47.455297+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455031+0000","last_change":"2023-10-09T06:52:41.266379+0000","last_active":"2023-10-09T06:54:47.455031+0000","last_peered":"2023-10-09T06:54:47.455031+0000","last_clean":"2023-10-09T06:54:47.455031+0000","last_became_active":"2023-10-09T06:52:41.262339+0000","last_became_peered":"2023-10-09T06:52:41.262339+0000","last_unstale":"2023-10-09T06:54:47.455031+0000","last_undegraded":"2023-10-09T06:54:47.455031+0000","last_fullsized":"2023-10-09T06:54:47.455031+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":52,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455551+0000","last_change":"2023-10-09T06:51:41.326746+0000","last_active":"2023-10-09T06:54:47.455551+0000","last_peered":"2023-10-09T06:54:47.455551+0000","last_clean":"2023-10-09T06:54:47.455551+0000","last_became_active":"2023-10-09T06:51:41.235036+0000","last_became_peered":"2023-10-09T06:51:41.235036+0000","last_unstale":"2023-10-09T06:54:47.455551+0000","last_undegraded":"2023-10-09T06:54:47.455551+0000","last_fullsized":"2023-10-09T06:54:47.455551+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.455367+0000","last_change":"2023-10-09T06:51:08.218437+0000","last_active":"2023-10-09T06:54:47.455367+0000","last_peered":"2023-10-09T06:54:47.455367+0000","last_clean":"2023-10-09T06:54:47.455367+0000","last_became_active":"2023-10-09T06:51:08.218044+0000","last_became_peered":"2023-10-09T06:51:08.218044+0000","last_unstale":"2023-10-09T06:54:47.455367+0000","last_undegraded":"2023-10-09T06:54:47.455367+0000","last_fullsized":"2023-10-09T06:54:47.455367+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.670509+0000","last_change":"2023-10-09T06:51:53.658314+0000","last_active":"2023-10-09T06:54:56.670509+0000","last_peered":"2023-10-09T06:54:56.670509+0000","last_clean":"2023-10-09T06:54:56.670509+0000","last_became_active":"2023-10-09T06:51:53.656771+0000","last_became_peered":"2023-10-09T06:51:53.656771+0000","last_unstale":"2023-10-09T06:54:56.670509+0000","last_undegraded":"2023-10-09T06:54:56.670509+0000","last_fullsized":"2023-10-09T06:54:56.670509+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:21.614043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-10-09T06:54:56.669671+0000","last_change":"2023-10-09T06:51:53.657581+0000","last_active":"2023-10-09T06:54:56.669671+0000","last_peered":"2023-10-09T06:54:56.669671+0000","last_clean":"2023-10-09T06:54:56.669671+0000","last_became_active":"2023-10-09T06:51:53.656320+0000","last_became_peered":"2023-10-09T06:51:53.656320+0000","last_unstale":"2023-10-09T06:54:56.669671+0000","last_undegraded":"2023-10-09T06:54:56.669671+0000","last_fullsized":"2023-10-09T06:54:56.669671+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:42.694344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":361,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.450952+0000","last_change":"2023-10-09T06:52:29.658771+0000","last_active":"2023-10-09T06:54:47.450952+0000","last_peered":"2023-10-09T06:54:47.450952+0000","last_clean":"2023-10-09T06:54:47.450952+0000","last_became_active":"2023-10-09T06:52:29.658306+0000","last_became_peered":"2023-10-09T06:52:29.658306+0000","last_unstale":"2023-10-09T06:54:47.450952+0000","last_undegraded":"2023-10-09T06:54:47.450952+0000","last_fullsized":"2023-10-09T06:54:47.450952+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:49.610347+0000","last_clean_scrub_stamp":"2023-10-09T06:01:49.610347+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:20.760541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":460,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-10-09T06:54:47.454948+0000","last_change":"2023-10-09T06:52:41.266360+0000","last_active":"2023-10-09T06:54:47.454948+0000","last_peered":"2023-10-09T06:54:47.454948+0000","last_clean":"2023-10-09T06:54:47.454948+0000","last_became_active":"2023-10-09T06:52:41.261562+0000","last_became_peered":"2023-10-09T06:52:41.261562+0000","last_unstale":"2023-10-09T06:54:47.454948+0000","last_undegraded":"2023-10-09T06:54:47.454948+0000","last_fullsized":"2023-10-09T06:54:47.454948+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:01:50.622376+0000","last_clean_scrub_stamp":"2023-10-09T06:01:50.622376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":31644,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27718,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11898,"num_read_kb":25381,"num_write":1690,"num_write_kb":25426,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":16777368,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430905,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910212,"kb_used_data":4092,"kb_used_omap":0,"kb_used_meta":906112,"kb_avail":92834940,"statfs":{"total":95995035648,"available":95062978560,"internally_reserved":0,"allocated":4190208,"data_stored":3510378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":927858688},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.879,"15min":1.3560000000000001},"min":{"1min":0.56100000000000005,"5min":0.40899999999999997,"15min":0.224},"max":{"1min":1.28,"5min":1.3280000000000001,"15min":92.066999999999993},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.88100000000000001,"15min":1.377},"min":{"1min":0.66900000000000004,"5min":0.47199999999999998,"15min":0.27600000000000002},"max":{"1min":1.2350000000000001,"5min":1.4219999999999999,"15min":91.715999999999994},"last":0.98999999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.92200000000000004,"15min":1.2210000000000001},"min":{"1min":0.58499999999999996,"5min":0.45800000000000002,"15min":0.28699999999999998},"max":{"1min":1.3220000000000001,"5min":1.6299999999999999,"15min":13.526999999999999},"last":0.877},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.97999999999999998,"15min":1.1200000000000001},"min":{"1min":0.67100000000000004,"5min":0.39400000000000002,"15min":0.27200000000000002},"max":{"1min":1.347,"5min":1.5149999999999999,"15min":10.798999999999999},"last":1.3}]},{"osd":2,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.89300000000000002,"15min":1.329},"min":{"1min":0.63100000000000001,"5min":0.61699999999999999,"15min":0.29099999999999998},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":9.0679999999999996},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.0009999999999999,"15min":1.1399999999999999},"min":{"1min":0.748,"5min":0.65000000000000002,"15min":0.34699999999999998},"max":{"1min":1.2889999999999999,"5min":1.4710000000000001,"15min":6.3319999999999999},"last":1.1579999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.98799999999999999,"15min":1.649},"min":{"1min":0.69299999999999995,"5min":0.61899999999999999,"15min":0.36499999999999999},"max":{"1min":1.111,"5min":1.4079999999999999,"15min":23.074000000000002},"last":1.2},{"interface":"front","average":{"1min":1.119,"5min":1.0980000000000001,"15min":1.425},"min":{"1min":0.79100000000000004,"5min":0.73499999999999999,"15min":0.36699999999999999},"max":{"1min":1.4139999999999999,"5min":1.677,"15min":23.163},"last":1.3620000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.94099999999999995,"15min":1.1519999999999999},"min":{"1min":0.56999999999999995,"5min":0.43099999999999999,"15min":0.26700000000000002},"max":{"1min":1.149,"5min":1.4670000000000001,"15min":12.711},"last":1.2769999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.96899999999999997,"15min":1.5800000000000001},"min":{"1min":0.64600000000000002,"5min":0.53900000000000003,"15min":0.32700000000000001},"max":{"1min":1.2090000000000001,"5min":1.484,"15min":11.369999999999999},"last":1.1299999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0820000000000001,"15min":1.107},"min":{"1min":0.59199999999999997,"5min":0.53300000000000003,"15min":0.35099999999999998},"max":{"1min":1.4410000000000001,"5min":1.6240000000000001,"15min":6.3049999999999997},"last":1.409},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.069,"15min":1.0580000000000001},"min":{"1min":0.64400000000000002,"5min":0.61399999999999999,"15min":0.34100000000000003},"max":{"1min":1.3260000000000001,"5min":1.4650000000000001,"15min":12.675000000000001},"last":1.095}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.095,"15min":1.3480000000000001},"min":{"1min":0.61199999999999999,"5min":0.56499999999999995,"15min":0.32200000000000001},"max":{"1min":1.3600000000000001,"5min":1.5349999999999999,"15min":12.640000000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.095,"5min":1.127,"15min":1.6739999999999999},"min":{"1min":0.629,"5min":0.59899999999999998,"15min":0.40500000000000003},"max":{"1min":1.3049999999999999,"5min":1.611,"15min":14.613},"last":1.254}]}]},{"osd":6,"up_from":42,"seq":180388627137,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900596,"kb_used_data":8172,"kb_used_omap":0,"kb_used_meta":892416,"kb_avail":92844556,"statfs":{"total":95995035648,"available":95072825344,"internally_reserved":0,"allocated":8368128,"data_stored":7703925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913833984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.89400000000000002,"15min":2.1739999999999999},"min":{"1min":0.65100000000000002,"5min":0.34799999999999998,"15min":0.23799999999999999},"max":{"1min":1.264,"5min":1.5720000000000001,"15min":120.822},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.90600000000000003,"15min":2.3159999999999998},"min":{"1min":0.627,"5min":0.375,"15min":0.28899999999999998},"max":{"1min":1.1679999999999999,"5min":1.381,"15min":120.47199999999999},"last":1.115}]},{"osd":1,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.873,"15min":1.1850000000000001},"min":{"1min":0.60499999999999998,"5min":0.53000000000000003,"15min":0.33300000000000002},"max":{"1min":1.1819999999999999,"5min":1.464,"15min":7.7539999999999996},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.95299999999999996,"15min":1.1579999999999999},"min":{"1min":0.65800000000000003,"5min":0.44800000000000001,"15min":0.29999999999999999},"max":{"1min":1.2909999999999999,"5min":1.4379999999999999,"15min":7.04},"last":1.1990000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.95099999999999996,"15min":2.2490000000000001},"min":{"1min":0.73299999999999998,"5min":0.57199999999999995,"15min":0.30599999999999999},"max":{"1min":1.2729999999999999,"5min":1.3440000000000001,"15min":271.976},"last":1.26},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.93999999999999995,"15min":2.3170000000000002},"min":{"1min":0.64400000000000002,"5min":0.58499999999999996,"15min":0.316},"max":{"1min":1.4099999999999999,"5min":1.4099999999999999,"15min":271.863},"last":0.90700000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.99299999999999999,"15min":1.2929999999999999},"min":{"1min":0.629,"5min":0.57599999999999996,"15min":0.35799999999999998},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":22.873999999999999},"last":1.099},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.98399999999999999,"15min":1.486},"min":{"1min":0.68500000000000005,"5min":0.63800000000000001,"15min":0.35099999999999998},"max":{"1min":1.198,"5min":3.0209999999999999,"15min":25.986999999999998},"last":1.155}]},{"osd":4,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.93000000000000005,"15min":1.77},"min":{"1min":0.60699999999999998,"5min":0.42999999999999999,"15min":0.30199999999999999},"max":{"1min":1.278,"5min":1.3859999999999999,"15min":80.231999999999999},"last":1.1759999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.97399999999999998,"15min":1.6779999999999999},"min":{"1min":0.63500000000000001,"5min":0.504,"15min":0.28799999999999998},"max":{"1min":1.246,"5min":1.548,"15min":81.085999999999999},"last":0.93400000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:56:07 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.0529999999999999,"15min":1.218},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.34899999999999998},"max":{"1min":1.3340000000000001,"5min":1.4510000000000001,"15min":11.558},"last":0.95999999999999996},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0449999999999999,"15min":1.1359999999999999},"min":{"1min":0.72699999999999998,"5min":0.61899999999999999,"15min":0.311},"max":{"1min":1.292,"5min":1.643,"15min":5.7400000000000002},"last":1.0640000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:55:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.1000000000000001,"15min":1.619},"min":{"1min":0.82999999999999996,"5min":0.58499999999999996,"15min":0.36099999999999999},"max":{"1min":1.2769999999999999,"5min":1.5629999999999999,"15min":12.177},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.143,"5min":1.1359999999999999,"15min":1.2789999999999999},"min":{"1min":0.60199999999999998,"5min":0.41699999999999998,"15min":0.35399999999999998},"max":{"1min":1.347,"5min":1.5389999999999999,"15min":9.5619999999999994},"last":1.3160000000000001}]}]},{"osd":1,"up_from":93,"seq":399431959047,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":683572,"kb_used_data":8428,"kb_used_omap":7,"kb_used_meta":675128,"kb_avail":93061580,"statfs":{"total":95995035648,"available":95295057920,"internally_reserved":0,"allocated":8630272,"data_stored":7208150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":691331472},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.72999999999999998,"15min":1.0309999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.19600000000000001},"max":{"1min":1.028,"5min":5.3230000000000004,"15min":17.201000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.73199999999999998,"15min":0.97099999999999997},"min":{"1min":0.41499999999999998,"5min":0.29199999999999998,"15min":0.17899999999999999},"max":{"1min":0.94799999999999995,"5min":2.169,"15min":15.981999999999999},"last":0.63800000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.71799999999999997,"15min":1.111},"min":{"1min":0.36799999999999999,"5min":0.26500000000000001,"15min":0.20999999999999999},"max":{"1min":1.133,"5min":2.0760000000000001,"15min":16.004000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.82099999999999995,"15min":1.2749999999999999},"min":{"1min":0.45100000000000001,"5min":0.38900000000000001,"15min":0.22600000000000001},"max":{"1min":1.0589999999999999,"5min":1.331,"15min":16.971},"last":0.73999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.90100000000000002,"15min":1.4430000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.27400000000000002},"max":{"1min":1.2729999999999999,"5min":1.986,"15min":47.143000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.83599999999999997,"15min":1.2450000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.27500000000000002},"max":{"1min":1.0309999999999999,"5min":5.4260000000000002,"15min":46.826999999999998},"last":0.69599999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.083,"15min":1.4279999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.318},"max":{"1min":1.4910000000000001,"5min":1.863,"15min":28.626999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.026,"15min":1.262},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.317},"max":{"1min":1.1259999999999999,"5min":5.5949999999999998,"15min":30},"last":0.94399999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.038,"15min":1.012},"min":{"1min":0.66600000000000004,"5min":0.66000000000000003,"15min":0.38700000000000001},"max":{"1min":1.3360000000000001,"5min":4.3579999999999997,"15min":4.9000000000000004},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1599999999999999,"15min":1.1830000000000001},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.375},"max":{"1min":1.5389999999999999,"5min":4.2350000000000003,"15min":8.077},"last":1.0900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.081,"15min":1.2130000000000001},"min":{"1min":0.70599999999999996,"5min":0.67100000000000004,"15min":0.36899999999999999},"max":{"1min":1.431,"5min":4.5419999999999998,"15min":25.553000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.1879999999999999,"15min":1.4370000000000001},"min":{"1min":0.748,"5min":0.66600000000000004,"15min":0.40100000000000002},"max":{"1min":1.544,"5min":4.1340000000000003,"15min":25.675000000000001},"last":1.1220000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:55:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.216,"15min":1.4730000000000001},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.41899999999999998},"max":{"1min":1.4730000000000001,"5min":3.8759999999999999,"15min":25.585999999999999},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.1599999999999999,"15min":1.353},"min":{"1min":0.81799999999999995,"5min":0.69899999999999995,"15min":0.432},"max":{"1min":1.355,"5min":5.718,"15min":25.518000000000001},"last":1.103}]}]},{"osd":0,"up_from":88,"seq":377957122572,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":940480,"kb_used_data":8952,"kb_used_omap":8,"kb_used_meta":931511,"kb_avail":92804672,"statfs":{"total":95995035648,"available":95031984128,"internally_reserved":0,"allocated":9166848,"data_stored":7789639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":953867520},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:56:00 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.73399999999999999,"15min":1.026},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.20000000000000001},"max":{"1min":1.0680000000000001,"5min":1.921,"15min":10.449},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.82699999999999996,"15min":1.0800000000000001},"min":{"1min":0.57199999999999995,"5min":0.375,"15min":0.254},"max":{"1min":1.1659999999999999,"5min":1.3839999999999999,"15min":13.055999999999999},"last":0.95599999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.72199999999999998,"15min":1.036},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.20599999999999999},"max":{"1min":0.78500000000000003,"5min":1.9810000000000001,"15min":12.154},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.70299999999999996,"15min":0.96699999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.223},"max":{"1min":0.879,"5min":1.1759999999999999,"15min":12.448},"last":1.181}]},{"osd":3,"last update":"Mon Oct 9 06:56:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.70399999999999996,"15min":1.0509999999999999},"min":{"1min":0.42999999999999999,"5min":0.28799999999999998,"15min":0.23300000000000001},"max":{"1min":0.95699999999999996,"5min":1.2290000000000001,"15min":11.635999999999999},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.83699999999999997,"15min":1.224},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.25800000000000001},"max":{"1min":0.95499999999999996,"5min":1.425,"15min":15.545999999999999},"last":0.74199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93000000000000005,"15min":1.2210000000000001},"min":{"1min":0.65300000000000002,"5min":0.57199999999999995,"15min":0.32800000000000001},"max":{"1min":1.2949999999999999,"5min":1.4790000000000001,"15min":9.5459999999999994},"last":1.1439999999999999},{"interface":"front","average":{"1min":0.999,"5min":0.98899999999999999,"15min":1.22},"min":{"1min":0.59199999999999997,"5min":0.501,"15min":0.31},"max":{"1min":1.3540000000000001,"5min":2.056,"15min":11.948},"last":0.68999999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.016,"15min":1.1259999999999999},"min":{"1min":0.755,"5min":0.60499999999999998,"15min":0.36699999999999999},"max":{"1min":1.3009999999999999,"5min":2.02,"15min":11.894},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.077,"5min":1.095,"15min":1.1779999999999999},"min":{"1min":0.63,"5min":0.47799999999999998,"15min":0.40400000000000003},"max":{"1min":1.3300000000000001,"5min":1.6200000000000001,"15min":8.5630000000000006},"last":1.0509999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":1.109,"15min":1.379},"min":{"1min":0.85099999999999998,"5min":0.53700000000000003,"15min":0.36799999999999999},"max":{"1min":1.3939999999999999,"5min":1.5780000000000001,"15min":8.5269999999999992},"last":0.879},{"interface":"front","average":{"1min":1.069,"5min":1.0489999999999999,"15min":1.27},"min":{"1min":0.70599999999999996,"5min":0.51400000000000001,"15min":0.379},"max":{"1min":1.3220000000000001,"5min":1.764,"15min":6.5869999999999997},"last":1.228}]},{"osd":7,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1759999999999999,"15min":1.4450000000000001},"min":{"1min":0.875,"5min":0.64900000000000002,"15min":0.40100000000000002},"max":{"1min":1.4930000000000001,"5min":1.6839999999999999,"15min":8.4819999999999993},"last":1.117},{"interface":"front","average":{"1min":1.125,"5min":1.089,"15min":1.3640000000000001},"min":{"1min":0.65200000000000002,"5min":0.46000000000000002,"15min":0.39300000000000002},"max":{"1min":1.3460000000000001,"5min":1.659,"15min":6.6120000000000001},"last":1.3260000000000001}]}]},{"osd":2,"up_from":98,"seq":420906795521,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670316,"kb_used_data":8420,"kb_used_omap":11,"kb_used_meta":661876,"kb_avail":93074836,"statfs":{"total":95995035648,"available":95308632064,"internally_reserved":0,"allocated":8622080,"data_stored":7202150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11330,"internal_metadata":677761982},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.68200000000000005,"15min":1.111},"min":{"1min":0.311,"5min":0.311,"15min":0.185},"max":{"1min":0.86299999999999999,"5min":1.228,"15min":38.210000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.70899999999999996,"15min":1.0509999999999999},"min":{"1min":0.433,"5min":0.40200000000000002,"15min":0.219},"max":{"1min":1.244,"5min":1.244,"15min":39.896000000000001},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.74199999999999999,"15min":0.92100000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.223},"max":{"1min":1.024,"5min":1.292,"15min":11.757},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.75600000000000001,"15min":1.0429999999999999},"min":{"1min":0.39000000000000001,"5min":0.36199999999999999,"15min":0.223},"max":{"1min":0.95399999999999996,"5min":1.306,"15min":42.710000000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.75800000000000001,"15min":1.575},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.24399999999999999},"max":{"1min":0.96699999999999997,"5min":1.196,"15min":52.060000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.747,"5min":0.74299999999999999,"15min":1.502},"min":{"1min":0.44700000000000001,"5min":0.37,"15min":0.255},"max":{"1min":1.1279999999999999,"5min":1.349,"15min":51.880000000000003},"last":0.64800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.999,"15min":1.3089999999999999},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.33700000000000002},"max":{"1min":1.381,"5min":4.1920000000000002,"15min":43.244},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.99299999999999999,"15min":1.3},"min":{"1min":0.66700000000000004,"5min":0.65200000000000002,"15min":0.32100000000000001},"max":{"1min":1.085,"5min":4.2800000000000002,"15min":42.783999999999999},"last":0.89600000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.028,"15min":1.111},"min":{"1min":0.63400000000000001,"5min":0.57999999999999996,"15min":0.35999999999999999},"max":{"1min":1.3089999999999999,"5min":3.4039999999999999,"15min":43.210000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.034,"15min":1.196},"min":{"1min":0.69699999999999995,"5min":0.52800000000000002,"15min":0.371},"max":{"1min":1.403,"5min":3.601,"15min":42.737000000000002},"last":1.0189999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.042,"15min":1.296},"min":{"1min":0.749,"5min":0.68400000000000005,"15min":0.36199999999999999},"max":{"1min":1.2390000000000001,"5min":2.968,"15min":16.332999999999998},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.113,"15min":1.379},"min":{"1min":0.68799999999999994,"5min":0.65500000000000003,"15min":0.35799999999999998},"max":{"1min":1.2889999999999999,"5min":3.7989999999999999,"15min":42.834000000000003},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:56:29 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.155,"15min":1.5369999999999999},"min":{"1min":0.84199999999999997,"5min":0.61899999999999999,"15min":0.39000000000000001},"max":{"1min":1.353,"5min":3.2879999999999998,"15min":42.804000000000002},"last":0.93000000000000005},{"interface":"front","average":{"1min":1.048,"5min":1.141,"15min":1.5569999999999999},"min":{"1min":0.80400000000000005,"5min":0.51300000000000001,"15min":0.40200000000000002},"max":{"1min":1.429,"5min":3.3370000000000002,"15min":43.164000000000001},"last":0.84499999999999997}]}]},{"osd":3,"up_from":103,"seq":442381631995,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757920,"kb_used_data":8472,"kb_used_omap":3,"kb_used_meta":749436,"kb_avail":92987232,"statfs":{"total":95995035648,"available":95218925568,"internally_reserved":0,"allocated":8675328,"data_stored":7206641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":767423009},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.70999999999999996,"15min":1.131},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.20699999999999999},"max":{"1min":1.0640000000000001,"5min":1.2470000000000001,"15min":44.442},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.70099999999999996,"15min":1.181},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.218},"max":{"1min":0.86599999999999999,"5min":1.226,"15min":44.250999999999998},"last":0.82699999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.69499999999999995,"15min":1.0649999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.221},"max":{"1min":0.85599999999999998,"5min":1.0600000000000001,"15min":11.577999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.75900000000000001,"15min":1.0820000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.21299999999999999},"max":{"1min":1.0149999999999999,"5min":1.1140000000000001,"15min":10.305999999999999},"last":0.78100000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":1.5669999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.25700000000000001},"max":{"1min":1.238,"5min":1.238,"15min":166.89500000000001},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.79200000000000004,"15min":1.542},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.28100000000000003},"max":{"1min":1.278,"5min":1.3220000000000001,"15min":165.86600000000001},"last":0.86499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.94799999999999995,"15min":1.208},"min":{"1min":0.59799999999999998,"5min":0.46600000000000003,"15min":0.30299999999999999},"max":{"1min":1.351,"5min":1.48,"15min":11.673999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.01,"5min":0.99399999999999999,"15min":1.254},"min":{"1min":0.68799999999999994,"5min":0.54000000000000004,"15min":0.32800000000000001},"max":{"1min":1.429,"5min":1.5529999999999999,"15min":11.365},"last":1.2509999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.038,"15min":1.2150000000000001},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.35799999999999998},"max":{"1min":1.4850000000000001,"5min":1.5109999999999999,"15min":11.439},"last":1.3939999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.048,"15min":1.2170000000000001},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.33700000000000002},"max":{"1min":1.3959999999999999,"5min":1.528,"15min":11.763},"last":0.98499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.1120000000000001,"15min":1.1910000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.379},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":9.718},"last":1.3149999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.117,"15min":1.3859999999999999},"min":{"1min":0.73299999999999998,"5min":0.71299999999999997,"15min":0.39600000000000002},"max":{"1min":1.381,"5min":1.6000000000000001,"15min":15.233000000000001},"last":1.351}]},{"osd":7,"last update":"Mon Oct 9 06:56:01 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.1040000000000001,"15min":1.4079999999999999},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.35899999999999999},"max":{"1min":1.468,"5min":1.5920000000000001,"15min":11.731},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.143,"5min":1.155,"15min":1.4279999999999999},"min":{"1min":0.67200000000000004,"5min":0.65000000000000002,"15min":0.35799999999999998},"max":{"1min":1.536,"5min":1.536,"15min":10.843},"last":1.375}]}]},{"osd":4,"up_from":30,"seq":128849019603,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906876,"kb_used_data":7604,"kb_used_omap":0,"kb_used_meta":899264,"kb_avail":92838276,"statfs":{"total":95995035648,"available":95066394624,"internally_reserved":0,"allocated":7786496,"data_stored":7114244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920846336},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.98499999999999999,"15min":2.327},"min":{"1min":0.67600000000000005,"5min":0.55500000000000005,"15min":0.28199999999999997},"max":{"1min":1.399,"5min":1.399,"15min":91.683999999999997},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.006,"5min":1.014,"15min":2.4249999999999998},"min":{"1min":0.73199999999999998,"5min":0.64200000000000002,"15min":0.26000000000000001},"max":{"1min":1.234,"5min":1.381,"15min":91.653999999999996},"last":0.82199999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":0.96999999999999997,"15min":1.139},"min":{"1min":0.72799999999999998,"5min":0.59899999999999998,"15min":0.28499999999999998},"max":{"1min":1.2909999999999999,"5min":1.4550000000000001,"15min":7.0659999999999998},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":0.998,"15min":1.0549999999999999},"min":{"1min":0.76200000000000001,"5min":0.65600000000000003,"15min":0.33500000000000002},"max":{"1min":1.3660000000000001,"5min":1.546,"15min":5.992},"last":1}]},{"osd":2,"last update":"Mon Oct 9 06:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.89100000000000001,"15min":2.544},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.28799999999999998},"max":{"1min":1.22,"5min":1.335,"15min":231.393},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93899999999999995,"15min":2.3570000000000002},"min":{"1min":0.55800000000000005,"5min":0.55600000000000005,"15min":0.29999999999999999},"max":{"1min":1.214,"5min":1.637,"15min":231.17099999999999},"last":0.87}]},{"osd":3,"last update":"Mon Oct 9 06:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.98499999999999999,"15min":2.5249999999999999},"min":{"1min":0.80300000000000005,"5min":0.63600000000000001,"15min":0.34699999999999998},"max":{"1min":1.264,"5min":1.361,"15min":234.62200000000001},"last":1.095},{"interface":"front","average":{"1min":1.095,"5min":1.0389999999999999,"15min":2.6589999999999998},"min":{"1min":0.80000000000000004,"5min":0.55000000000000004,"15min":0.40400000000000003},"max":{"1min":1.256,"5min":1.51,"15min":234.673},"last":1.214}]},{"osd":5,"last update":"Mon Oct 9 06:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.94999999999999996,"15min":1.044},"min":{"1min":0.68600000000000005,"5min":0.57199999999999995,"15min":0.313},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":7.0330000000000004},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.97899999999999998,"15min":1.1679999999999999},"min":{"1min":0.73099999999999998,"5min":0.52900000000000003,"15min":0.27700000000000002},"max":{"1min":1.321,"5min":1.522,"15min":10.707000000000001},"last":1.0389999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0409999999999999,"15min":1.5289999999999999},"min":{"1min":0.83499999999999996,"5min":0.53000000000000003,"15min":0.38100000000000001},"max":{"1min":1.2709999999999999,"5min":1.8129999999999999,"15min":13.163},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0680000000000001,"15min":1.337},"min":{"1min":0.78900000000000003,"5min":0.64000000000000001,"15min":0.373},"max":{"1min":1.3720000000000001,"5min":2.137,"15min":10.138999999999999},"last":1.1479999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.093,"15min":1.6990000000000001},"min":{"1min":0.84499999999999997,"5min":0.629,"15min":0.312},"max":{"1min":1.3959999999999999,"5min":1.468,"15min":10.688000000000001},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0569999999999999,"15min":1.1779999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.35599999999999998},"max":{"1min":1.335,"5min":1.476,"15min":8.9529999999999994},"last":1.167}]}]},{"osd":5,"up_from":36,"seq":154618823370,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":597292,"kb_used_data":3492,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93147860,"statfs":{"total":95995035648,"available":95383408640,"internally_reserved":0,"allocated":3575808,"data_stored":2915346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.94199999999999995,"15min":2.762},"min":{"1min":0.59299999999999997,"5min":0.502,"15min":0.25800000000000001},"max":{"1min":1.2749999999999999,"5min":1.948,"15min":309.678},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.91300000000000003,"15min":3.028},"min":{"1min":0.59399999999999997,"5min":0.56100000000000005,"15min":0.29299999999999998},"max":{"1min":1.2789999999999999,"5min":1.3620000000000001,"15min":304.83300000000003},"last":0.69499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:56:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.88200000000000001,"15min":0.93999999999999995},"min":{"1min":0.55000000000000004,"5min":0.52500000000000002,"15min":0.309},"max":{"1min":1.2230000000000001,"5min":1.28,"15min":6.7080000000000002},"last":1.034},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.90200000000000002,"15min":0.92200000000000004},"min":{"1min":0.628,"5min":0.48099999999999998,"15min":0.28299999999999997},"max":{"1min":1.2030000000000001,"5min":1.744,"15min":7.3330000000000002},"last":0.82399999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.94999999999999996,"15min":1.647},"min":{"1min":0.71599999999999997,"5min":0.497,"15min":0.34100000000000003},"max":{"1min":1.2729999999999999,"5min":2.6429999999999998,"15min":187.732},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.97399999999999998,"15min":1.6970000000000001},"min":{"1min":0.72299999999999998,"5min":0.57799999999999996,"15min":0.33500000000000002},"max":{"1min":1.206,"5min":1.3819999999999999,"15min":188.286},"last":0.93999999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.99299999999999999,"15min":1.4199999999999999},"min":{"1min":0.65000000000000002,"5min":0.58799999999999997,"15min":0.34699999999999998},"max":{"1min":1.234,"5min":2.7290000000000001,"15min":48.509},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.036,"15min":1.371},"min":{"1min":0.90500000000000003,"5min":0.70799999999999996,"15min":0.377},"max":{"1min":1.2829999999999999,"5min":1.47,"15min":48.328000000000003},"last":1.099}]},{"osd":4,"last update":"Mon Oct 9 06:56:18 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.91300000000000003,"15min":1.155},"min":{"1min":0.71499999999999997,"5min":0.46300000000000002,"15min":0.33100000000000002},"max":{"1min":1.3859999999999999,"5min":2.0019999999999998,"15min":43.087000000000003},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0469999999999999,"15min":1.248},"min":{"1min":0.73999999999999999,"5min":0.496,"15min":0.34899999999999998},"max":{"1min":1.2929999999999999,"5min":3.3370000000000002,"15min":42.713000000000001},"last":1.111}]},{"osd":6,"last update":"Mon Oct 9 06:56:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.99099999999999999,"15min":1.036},"min":{"1min":0.72699999999999998,"5min":0.51600000000000001,"15min":0.36399999999999999},"max":{"1min":1.476,"5min":1.8220000000000001,"15min":7.3769999999999998},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.131,"15min":1.2749999999999999},"min":{"1min":0.77500000000000002,"5min":0.53800000000000003,"15min":0.371},"max":{"1min":1.349,"5min":2.8570000000000002,"15min":12.252000000000001},"last":1.1910000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:56:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1679999999999999,"15min":1.212},"min":{"1min":0.88200000000000001,"5min":0.56899999999999995,"15min":0.379},"max":{"1min":1.4670000000000001,"5min":2.7949999999999999,"15min":9.2200000000000006},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.127,"15min":1.119},"min":{"1min":0.98099999999999998,"5min":0.57799999999999996,"15min":0.35199999999999998},"max":{"1min":1.349,"5min":1.5580000000000001,"15min":9.1950000000000003},"last":1.139}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3915,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8960,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7792,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3551,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:56:43.771 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:56:43.771 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:56:43.848 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=a433327857fe46cc922ffa60c80bdf7d Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:56:43.851 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T06:56:43.864 INFO:tasks.cephadm:Teardown begin 2023-10-09T06:56:43.865 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:56:43.885 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:56:43.900 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T06:56:43.900 DEBUG:teuthology.orchestra.run.smithi125:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:56:43.938 DEBUG:teuthology.orchestra.run.smithi155:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:56:43.948 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T06:56:43.949 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T06:56:43.949 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T06:56:44.020 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:43 smithi125 systemd[1]: Stopping Ceph mon.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:44.369 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[80002]: audit 2023-10-09T06:56:42.718857+0000 mgr.y (mgr.25723) 1573 : audit 0 from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:44.370 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE Bus STOPPING 2023-10-09T06:56:44.370 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:56:44.370 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE Bus STOPPED 2023-10-09T06:56:44.371 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE Bus STARTING 2023-10-09T06:56:44.371 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE Serving on http://:::9283 2023-10-09T06:56:44.371 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[77299]: audit 2023-10-09T06:56:42.718857+0000 mgr.y (mgr.25723) 1573 : audit 0 from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:44.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[77299]: debug 2023-10-09T06:56:44.232+0000 7ff94f7e4700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:56:44.372 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[77299]: debug 2023-10-09T06:56:44.232+0000 7ff94f7e4700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T06:56:44.394 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:44 smithi155 bash[59495]: audit 2023-10-09T06:56:42.718857+0000 mgr.y (mgr.25723) 1573 : audit 0 from='client.47033 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:56:44.636 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:44 smithi125 bash[73481]: [09/Oct/2023:06:56:44] ENGINE Bus STARTED 2023-10-09T06:56:45.605 INFO:journalctl@ceph.mon.a.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[263725]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-a 2023-10-09T06:56:45.619 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T06:56:45.728 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:56:45.729 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T06:56:45.731 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T06:56:45.733 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T06:56:45.886 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:45 smithi125 systemd[1]: Stopping Ceph mon.c for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:46.185 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[80002]: debug 2023-10-09T06:56:45.900+0000 7fb869ac2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:56:46.185 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[80002]: debug 2023-10-09T06:56:45.900+0000 7fb869ac2700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:56:46.185 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[73481]: [09/Oct/2023:06:56:45] ENGINE Bus STOPPING 2023-10-09T06:56:46.185 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[73481]: [09/Oct/2023:06:56:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:56:46.186 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[73481]: [09/Oct/2023:06:56:45] ENGINE Bus STOPPED 2023-10-09T06:56:46.186 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:45 smithi125 bash[73481]: [09/Oct/2023:06:56:45] ENGINE Bus STARTING 2023-10-09T06:56:46.186 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:46 smithi125 bash[73481]: [09/Oct/2023:06:56:46] ENGINE Serving on http://:::9283 2023-10-09T06:56:46.186 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:46 smithi125 bash[73481]: [09/Oct/2023:06:56:46] ENGINE Bus STARTED 2023-10-09T06:56:47.215 INFO:journalctl@ceph.mon.c.smithi125.stdout:Oct 09 06:56:46 smithi125 bash[263840]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-c 2023-10-09T06:56:47.221 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T06:56:47.258 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:56:47.259 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T06:56:47.259 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T06:56:47.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T06:56:47.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:47 smithi155 systemd[1]: Stopping Ceph mon.b for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:47.530 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:47 smithi155 bash[59495]: debug 2023-10-09T06:56:47.480+0000 7fd7e0304700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-09T06:56:47.531 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:47 smithi155 bash[59495]: debug 2023-10-09T06:56:47.480+0000 7fd7e0304700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:56:48.534 INFO:journalctl@ceph.mon.b.smithi155.stdout:Oct 09 06:56:48 smithi155 bash[158350]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mon-b 2023-10-09T06:56:48.540 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T06:56:48.582 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:56:48.582 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T06:56:48.582 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T06:56:48.582 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T06:56:48.854 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:48 smithi125 systemd[1]: Stopping Ceph mgr.y for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:48.854 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:48 smithi125 bash[73481]: debug 2023-10-09T06:56:48.756+0000 7f84d3eab700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:56:49.790 INFO:journalctl@ceph.mgr.y.smithi125.stdout:Oct 09 06:56:49 smithi125 bash[263953]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-y 2023-10-09T06:56:49.796 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T06:56:49.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:56:49.833 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T06:56:49.833 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T06:56:49.834 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T06:56:50.100 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:56:49 smithi155 systemd[1]: Stopping Ceph mgr.x for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:51.002 INFO:journalctl@ceph.mgr.x.smithi155.stdout:Oct 09 06:56:50 smithi155 bash[158463]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-mgr-x 2023-10-09T06:56:51.021 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T06:56:51.052 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:56:51.052 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T06:56:51.052 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T06:56:51.053 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:56:51.386 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:51 smithi125 systemd[1]: Stopping Ceph osd.0 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:56:51.387 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:51 smithi125 bash[85861]: debug 2023-10-09T06:56:51.224+0000 7fc8f8f07700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:56:51.387 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:51 smithi125 bash[85861]: debug 2023-10-09T06:56:51.224+0000 7fc8f8f07700 -1 osd.0 406 *** Got signal Terminated *** 2023-10-09T06:56:51.387 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:51 smithi125 bash[85861]: debug 2023-10-09T06:56:51.224+0000 7fc8f8f07700 -1 osd.0 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:56:57.824 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:57 smithi125 bash[264057]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T06:56:58.136 INFO:journalctl@ceph.osd.0.smithi125.stdout:Oct 09 06:56:57 smithi125 bash[264123]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-0 2023-10-09T06:57:00.470 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T06:57:00.504 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:00.505 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T06:57:00.505 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T06:57:00.505 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T06:57:00.722 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:00 smithi125 systemd[1]: Stopping Ceph osd.1 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:01.136 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:00 smithi125 bash[88651]: debug 2023-10-09T06:57:00.716+0000 7f7934f84700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:01.136 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:00 smithi125 bash[88651]: debug 2023-10-09T06:57:00.716+0000 7f7934f84700 -1 osd.1 406 *** Got signal Terminated *** 2023-10-09T06:57:01.137 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:00 smithi125 bash[88651]: debug 2023-10-09T06:57:00.716+0000 7f7934f84700 -1 osd.1 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:07.386 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:07 smithi125 bash[264266]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T06:57:07.386 INFO:journalctl@ceph.osd.1.smithi125.stdout:Oct 09 06:57:07 smithi125 bash[264334]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-1 2023-10-09T06:57:09.433 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T06:57:09.463 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:09.464 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T06:57:09.464 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T06:57:09.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T06:57:09.686 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:09 smithi125 systemd[1]: Stopping Ceph osd.2 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:10.136 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:09 smithi125 bash[91497]: debug 2023-10-09T06:57:09.679+0000 7f4127012700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:10.136 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:09 smithi125 bash[91497]: debug 2023-10-09T06:57:09.679+0000 7f4127012700 -1 osd.2 406 *** Got signal Terminated *** 2023-10-09T06:57:10.137 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:09 smithi125 bash[91497]: debug 2023-10-09T06:57:09.679+0000 7f4127012700 -1 osd.2 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:16.366 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:16 smithi125 bash[264473]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T06:57:16.366 INFO:journalctl@ceph.osd.2.smithi125.stdout:Oct 09 06:57:16 smithi125 bash[264539]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-2 2023-10-09T06:57:17.893 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:17 smithi155 bash[43742]: debug 2023-10-09T06:57:17.604+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:18.416 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T06:57:18.448 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:18.448 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T06:57:18.448 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T06:57:18.449 DEBUG:teuthology.orchestra.run.smithi125:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T06:57:18.670 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:18 smithi125 systemd[1]: Stopping Ceph osd.3 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:19.136 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:18 smithi125 bash[94382]: debug 2023-10-09T06:57:18.663+0000 7f73b34aa700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:19.136 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:18 smithi125 bash[94382]: debug 2023-10-09T06:57:18.663+0000 7f73b34aa700 -1 osd.3 406 *** Got signal Terminated *** 2023-10-09T06:57:19.137 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:18 smithi125 bash[94382]: debug 2023-10-09T06:57:18.663+0000 7f73b34aa700 -1 osd.3 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:19.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:18 smithi155 bash[43742]: debug 2023-10-09T06:57:18.656+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:19.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:18 smithi155 bash[31429]: debug 2023-10-09T06:57:18.740+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:19.971 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:19 smithi155 bash[31429]: debug 2023-10-09T06:57:19.708+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:19.971 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:19 smithi155 bash[43742]: debug 2023-10-09T06:57:19.648+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:19.972 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:19 smithi155 bash[39663]: debug 2023-10-09T06:57:19.772+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:20.393 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:19 smithi155 bash[35578]: debug 2023-10-09T06:57:19.968+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:20.636 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:20 smithi125 bash[94382]: debug 2023-10-09T06:57:20.339+0000 7f73ada9c700 -1 osd.3 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.342013+0000 front 2023-10-09T06:56:54.342335+0000 (oldest deadline 2023-10-09T06:57:20.241851+0000) 2023-10-09T06:57:21.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:20 smithi155 bash[39663]: debug 2023-10-09T06:57:20.796+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:21.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:20 smithi155 bash[31429]: debug 2023-10-09T06:57:20.748+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:21.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:20 smithi155 bash[35578]: debug 2023-10-09T06:57:20.920+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:21.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:20 smithi155 bash[43742]: debug 2023-10-09T06:57:20.672+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:21.636 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:21 smithi125 bash[94382]: debug 2023-10-09T06:57:21.347+0000 7f73ada9c700 -1 osd.3 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.342013+0000 front 2023-10-09T06:56:54.342335+0000 (oldest deadline 2023-10-09T06:57:20.241851+0000) 2023-10-09T06:57:22.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:21 smithi155 bash[35578]: debug 2023-10-09T06:57:21.920+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:22.149 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:21 smithi155 bash[31429]: debug 2023-10-09T06:57:21.788+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:22.149 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:21 smithi155 bash[43742]: debug 2023-10-09T06:57:21.696+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:22.150 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:21 smithi155 bash[39663]: debug 2023-10-09T06:57:21.760+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:22.887 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:22 smithi125 bash[94382]: debug 2023-10-09T06:57:22.387+0000 7f73ada9c700 -1 osd.3 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.342013+0000 front 2023-10-09T06:56:54.342335+0000 (oldest deadline 2023-10-09T06:57:20.241851+0000) 2023-10-09T06:57:23.143 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:22 smithi155 bash[35578]: debug 2023-10-09T06:57:22.920+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:23.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:22 smithi155 bash[31429]: debug 2023-10-09T06:57:22.788+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:23.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:22 smithi155 bash[39663]: debug 2023-10-09T06:57:22.792+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:23.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:22 smithi155 bash[43742]: debug 2023-10-09T06:57:22.720+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:23.886 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:23 smithi125 bash[94382]: debug 2023-10-09T06:57:23.431+0000 7f73ada9c700 -1 osd.3 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.342013+0000 front 2023-10-09T06:56:54.342335+0000 (oldest deadline 2023-10-09T06:57:20.241851+0000) 2023-10-09T06:57:24.143 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:23 smithi155 bash[31429]: debug 2023-10-09T06:57:23.796+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:24.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:23 smithi155 bash[35578]: debug 2023-10-09T06:57:23.880+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:24.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:23 smithi155 bash[39663]: debug 2023-10-09T06:57:23.824+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:24.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:23 smithi155 bash[43742]: debug 2023-10-09T06:57:23.676+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:25.143 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:24 smithi155 bash[35578]: debug 2023-10-09T06:57:24.888+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:25.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:24 smithi155 bash[39663]: debug 2023-10-09T06:57:24.856+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:25.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:24 smithi155 bash[31429]: debug 2023-10-09T06:57:24.796+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:25.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:24 smithi155 bash[43742]: debug 2023-10-09T06:57:24.664+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:25.206 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:24 smithi125 bash[264680]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T06:57:25.206 INFO:journalctl@ceph.osd.3.smithi125.stdout:Oct 09 06:57:25 smithi125 bash[264748]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-3 2023-10-09T06:57:26.143 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:25 smithi155 bash[35578]: debug 2023-10-09T06:57:25.895+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:26.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:25 smithi155 bash[39663]: debug 2023-10-09T06:57:25.883+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:26.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:25 smithi155 bash[31429]: debug 2023-10-09T06:57:25.815+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:26.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:25 smithi155 bash[43742]: debug 2023-10-09T06:57:25.687+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:27.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[39663]: debug 2023-10-09T06:57:26.907+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:27.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[39663]: debug 2023-10-09T06:57:26.907+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:27.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[43742]: debug 2023-10-09T06:57:26.663+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:27.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[31429]: debug 2023-10-09T06:57:26.863+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:27.145 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[31429]: debug 2023-10-09T06:57:26.863+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:27.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:26 smithi155 bash[35578]: debug 2023-10-09T06:57:26.855+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:27.215 DEBUG:teuthology.orchestra.run.smithi125:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T06:57:27.251 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:27.251 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T06:57:27.251 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T06:57:27.251 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:57:27.643 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 systemd[1]: Stopping Ceph osd.4 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:27.644 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[31429]: debug 2023-10-09T06:57:27.455+0000 7f1cf8425700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:27.644 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[31429]: debug 2023-10-09T06:57:27.455+0000 7f1cf8425700 -1 osd.4 406 *** Got signal Terminated *** 2023-10-09T06:57:27.644 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[31429]: debug 2023-10-09T06:57:27.455+0000 7f1cf8425700 -1 osd.4 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:27.948 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[31429]: debug 2023-10-09T06:57:27.867+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:27.948 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[31429]: debug 2023-10-09T06:57:27.867+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:27.948 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[43742]: debug 2023-10-09T06:57:27.675+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:27.948 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[43742]: debug 2023-10-09T06:57:27.675+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:27.949 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[35578]: debug 2023-10-09T06:57:27.851+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:28.393 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[39663]: debug 2023-10-09T06:57:27.943+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:28.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:27 smithi155 bash[39663]: debug 2023-10-09T06:57:27.943+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:28.921 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[35578]: debug 2023-10-09T06:57:28.843+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:28.921 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[35578]: debug 2023-10-09T06:57:28.843+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:28.921 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[43742]: debug 2023-10-09T06:57:28.655+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:28.922 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[43742]: debug 2023-10-09T06:57:28.655+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:29.393 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[31429]: debug 2023-10-09T06:57:28.915+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:29.394 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[31429]: debug 2023-10-09T06:57:28.915+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:29.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[39663]: debug 2023-10-09T06:57:28.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:29.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:28 smithi155 bash[39663]: debug 2023-10-09T06:57:28.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:29.973 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[31429]: debug 2023-10-09T06:57:29.919+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:29.973 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[31429]: debug 2023-10-09T06:57:29.919+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:29.974 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[43742]: debug 2023-10-09T06:57:29.695+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:29.974 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[43742]: debug 2023-10-09T06:57:29.695+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:29.974 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[35578]: debug 2023-10-09T06:57:29.871+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:29.974 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[35578]: debug 2023-10-09T06:57:29.871+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:30.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[39663]: debug 2023-10-09T06:57:29.967+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:30.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:29 smithi155 bash[39663]: debug 2023-10-09T06:57:29.967+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:31.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[39663]: debug 2023-10-09T06:57:30.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:31.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[39663]: debug 2023-10-09T06:57:30.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:31.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[31429]: debug 2023-10-09T06:57:30.875+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:31.144 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[31429]: debug 2023-10-09T06:57:30.875+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:31.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[35578]: debug 2023-10-09T06:57:30.919+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:31.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[35578]: debug 2023-10-09T06:57:30.919+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:31.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[43742]: debug 2023-10-09T06:57:30.727+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:31.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:30 smithi155 bash[43742]: debug 2023-10-09T06:57:30.727+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:31.969 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[31429]: debug 2023-10-09T06:57:31.915+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:56.027730+0000 front 2023-10-09T06:56:56.027712+0000 (oldest deadline 2023-10-09T06:57:18.327231+0000) 2023-10-09T06:57:31.970 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[31429]: debug 2023-10-09T06:57:31.915+0000 7f1cf2b9d700 -1 osd.4 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.230491+0000 front 2023-10-09T06:57:05.230429+0000 (oldest deadline 2023-10-09T06:57:26.330184+0000) 2023-10-09T06:57:31.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[43742]: debug 2023-10-09T06:57:31.703+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:31.970 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[43742]: debug 2023-10-09T06:57:31.703+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:32.393 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[35578]: debug 2023-10-09T06:57:31.963+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:32.394 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[35578]: debug 2023-10-09T06:57:31.963+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:32.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[39663]: debug 2023-10-09T06:57:31.967+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:32.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:31 smithi155 bash[39663]: debug 2023-10-09T06:57:31.967+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:33.052 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:32 smithi155 bash[43742]: debug 2023-10-09T06:57:32.731+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:33.052 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:32 smithi155 bash[43742]: debug 2023-10-09T06:57:32.731+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:33.394 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[35578]: debug 2023-10-09T06:57:32.931+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:33.394 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[35578]: debug 2023-10-09T06:57:32.931+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:33.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[39663]: debug 2023-10-09T06:57:32.923+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:33.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[39663]: debug 2023-10-09T06:57:32.923+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:33.893 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[158568]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T06:57:33.894 INFO:journalctl@ceph.osd.4.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[158633]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-4 2023-10-09T06:57:33.894 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[43742]: debug 2023-10-09T06:57:33.727+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:33.894 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[43742]: debug 2023-10-09T06:57:33.727+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:34.393 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[35578]: debug 2023-10-09T06:57:33.927+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:34.394 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[35578]: debug 2023-10-09T06:57:33.927+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:34.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[39663]: debug 2023-10-09T06:57:33.927+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:34.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:33 smithi155 bash[39663]: debug 2023-10-09T06:57:33.927+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:34.965 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[43742]: debug 2023-10-09T06:57:34.755+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:34.966 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[43742]: debug 2023-10-09T06:57:34.755+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:34.966 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[35578]: debug 2023-10-09T06:57:34.883+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:34.966 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[35578]: debug 2023-10-09T06:57:34.883+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:35.393 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[39663]: debug 2023-10-09T06:57:34.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:35.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:34 smithi155 bash[39663]: debug 2023-10-09T06:57:34.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:35.917 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:35 smithi155 bash[43742]: debug 2023-10-09T06:57:35.775+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:35.918 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:35 smithi155 bash[43742]: debug 2023-10-09T06:57:35.775+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:36.143 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:35 smithi155 bash[35578]: debug 2023-10-09T06:57:35.911+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:36.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:35 smithi155 bash[35578]: debug 2023-10-09T06:57:35.911+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:36.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[39663]: debug 2023-10-09T06:57:36.007+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:36.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[39663]: debug 2023-10-09T06:57:36.007+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:36.204 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T06:57:36.234 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:36.234 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T06:57:36.234 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T06:57:36.235 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:57:36.472 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 systemd[1]: Stopping Ceph osd.5 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:36.808 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[35578]: debug 2023-10-09T06:57:36.467+0000 7fd7eabfe700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:36.808 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[35578]: debug 2023-10-09T06:57:36.467+0000 7fd7eabfe700 -1 osd.5 406 *** Got signal Terminated *** 2023-10-09T06:57:36.808 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[35578]: debug 2023-10-09T06:57:36.467+0000 7fd7eabfe700 -1 osd.5 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:37.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[35578]: debug 2023-10-09T06:57:36.915+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:37.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[35578]: debug 2023-10-09T06:57:36.915+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:37.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.015+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:37.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.015+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:37.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.015+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:37.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[43742]: debug 2023-10-09T06:57:36.803+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:37.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[43742]: debug 2023-10-09T06:57:36.803+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:37.145 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:36 smithi155 bash[43742]: debug 2023-10-09T06:57:36.803+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:38.143 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[35578]: debug 2023-10-09T06:57:37.871+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:38.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[35578]: debug 2023-10-09T06:57:37.871+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:38.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[43742]: debug 2023-10-09T06:57:37.811+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:38.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[43742]: debug 2023-10-09T06:57:37.811+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:38.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[43742]: debug 2023-10-09T06:57:37.811+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:38.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:38.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:38.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:37 smithi155 bash[39663]: debug 2023-10-09T06:57:37.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:39.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[43742]: debug 2023-10-09T06:57:38.827+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:39.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[43742]: debug 2023-10-09T06:57:38.827+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:39.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[43742]: debug 2023-10-09T06:57:38.827+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:39.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[35578]: debug 2023-10-09T06:57:38.919+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:39.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[35578]: debug 2023-10-09T06:57:38.919+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:39.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[35578]: debug 2023-10-09T06:57:38.919+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.960483+0000 front 2023-10-09T06:57:13.960456+0000 (oldest deadline 2023-10-09T06:57:38.660088+0000) 2023-10-09T06:57:39.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[39663]: debug 2023-10-09T06:57:38.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:39.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[39663]: debug 2023-10-09T06:57:38.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:39.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:38 smithi155 bash[39663]: debug 2023-10-09T06:57:38.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:40.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[43742]: debug 2023-10-09T06:57:39.851+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:40.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[43742]: debug 2023-10-09T06:57:39.851+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:40.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[43742]: debug 2023-10-09T06:57:39.851+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:40.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[35578]: debug 2023-10-09T06:57:39.907+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:40.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[35578]: debug 2023-10-09T06:57:39.907+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:40.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[35578]: debug 2023-10-09T06:57:39.907+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.960483+0000 front 2023-10-09T06:57:13.960456+0000 (oldest deadline 2023-10-09T06:57:38.660088+0000) 2023-10-09T06:57:40.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[39663]: debug 2023-10-09T06:57:39.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:40.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[39663]: debug 2023-10-09T06:57:39.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:40.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:39 smithi155 bash[39663]: debug 2023-10-09T06:57:39.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:40.145 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[54956]: ts=2023-10-09T06:57:40.068Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T06:57:40.146 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[54956]: ts=2023-10-09T06:57:40.072Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T06:57:40.146 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[54956]: ts=2023-10-09T06:57:40.072Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T06:57:40.146 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[54956]: ts=2023-10-09T06:57:40.072Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.125:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.125:8765: connect: connection refused" 2023-10-09T06:57:41.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[43742]: debug 2023-10-09T06:57:40.863+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:41.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[43742]: debug 2023-10-09T06:57:40.863+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:41.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[43742]: debug 2023-10-09T06:57:40.863+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:41.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[35578]: debug 2023-10-09T06:57:40.939+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:54.557279+0000 front 2023-10-09T06:56:54.557215+0000 (oldest deadline 2023-10-09T06:57:19.257411+0000) 2023-10-09T06:57:41.144 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[35578]: debug 2023-10-09T06:57:40.939+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:03.959162+0000 front 2023-10-09T06:57:03.959094+0000 (oldest deadline 2023-10-09T06:57:28.658883+0000) 2023-10-09T06:57:41.145 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[35578]: debug 2023-10-09T06:57:40.939+0000 7fd7e3b73700 -1 osd.5 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.960483+0000 front 2023-10-09T06:57:13.960456+0000 (oldest deadline 2023-10-09T06:57:38.660088+0000) 2023-10-09T06:57:41.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[39663]: debug 2023-10-09T06:57:40.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:41.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[39663]: debug 2023-10-09T06:57:40.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:41.145 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:40 smithi155 bash[39663]: debug 2023-10-09T06:57:40.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:41.963 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[43742]: debug 2023-10-09T06:57:41.899+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:41.963 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[43742]: debug 2023-10-09T06:57:41.899+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:41.963 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[43742]: debug 2023-10-09T06:57:41.899+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:42.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[39663]: debug 2023-10-09T06:57:41.955+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:42.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[39663]: debug 2023-10-09T06:57:41.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:42.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:41 smithi155 bash[39663]: debug 2023-10-09T06:57:41.959+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:42.815 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[158774]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T06:57:42.815 INFO:journalctl@ceph.osd.5.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[158836]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-5 2023-10-09T06:57:43.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[39663]: debug 2023-10-09T06:57:42.931+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:43.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[39663]: debug 2023-10-09T06:57:42.931+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:43.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[39663]: debug 2023-10-09T06:57:42.931+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:43.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[43742]: debug 2023-10-09T06:57:42.935+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:43.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[43742]: debug 2023-10-09T06:57:42.935+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:43.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:42 smithi155 bash[43742]: debug 2023-10-09T06:57:42.935+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:44.143 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[39663]: debug 2023-10-09T06:57:43.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:44.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[39663]: debug 2023-10-09T06:57:43.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:44.144 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[39663]: debug 2023-10-09T06:57:43.947+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:44.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[43742]: debug 2023-10-09T06:57:43.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:44.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[43742]: debug 2023-10-09T06:57:43.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:44.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:43 smithi155 bash[43742]: debug 2023-10-09T06:57:43.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:45.115 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T06:57:45.147 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:45.148 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T06:57:45.148 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T06:57:45.148 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:57:45.203 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[39663]: debug 2023-10-09T06:57:44.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:45.203 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[39663]: debug 2023-10-09T06:57:44.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:45.203 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[39663]: debug 2023-10-09T06:57:44.951+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:45.203 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[43742]: debug 2023-10-09T06:57:44.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:45.203 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[43742]: debug 2023-10-09T06:57:44.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:45.204 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[43742]: debug 2023-10-09T06:57:44.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:45.204 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:44 smithi155 bash[43742]: debug 2023-10-09T06:57:44.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:45.643 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 systemd[1]: Stopping Ceph osd.6 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:45.644 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.395+0000 7ff4f04eb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:45.644 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.395+0000 7ff4f04eb700 -1 osd.6 406 *** Got signal Terminated *** 2023-10-09T06:57:45.644 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.395+0000 7ff4f04eb700 -1 osd.6 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:46.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[43742]: debug 2023-10-09T06:57:45.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:46.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[43742]: debug 2023-10-09T06:57:45.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:46.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[43742]: debug 2023-10-09T06:57:45.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:46.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[43742]: debug 2023-10-09T06:57:45.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:46.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:46.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:46.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:46.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:45 smithi155 bash[39663]: debug 2023-10-09T06:57:45.983+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:20.732312+0000 front 2023-10-09T06:57:20.732302+0000 (oldest deadline 2023-10-09T06:57:45.432137+0000) 2023-10-09T06:57:47.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.023+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:47.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.023+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:47.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.023+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:47.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.023+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:47.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:46 smithi155 bash[39663]: debug 2023-10-09T06:57:46.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:47.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:46 smithi155 bash[39663]: debug 2023-10-09T06:57:46.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:47.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:46 smithi155 bash[39663]: debug 2023-10-09T06:57:46.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:47.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:46 smithi155 bash[39663]: debug 2023-10-09T06:57:46.963+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:20.732312+0000 front 2023-10-09T06:57:20.732302+0000 (oldest deadline 2023-10-09T06:57:45.432137+0000) 2023-10-09T06:57:48.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:48.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:48.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:48.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:47 smithi155 bash[43742]: debug 2023-10-09T06:57:47.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:48.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[39663]: debug 2023-10-09T06:57:47.999+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:48.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[39663]: debug 2023-10-09T06:57:47.999+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:48.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[39663]: debug 2023-10-09T06:57:47.999+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:48.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[39663]: debug 2023-10-09T06:57:47.999+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:20.732312+0000 front 2023-10-09T06:57:20.732302+0000 (oldest deadline 2023-10-09T06:57:45.432137+0000) 2023-10-09T06:57:49.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[43742]: debug 2023-10-09T06:57:48.991+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:49.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[43742]: debug 2023-10-09T06:57:48.991+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:49.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[43742]: debug 2023-10-09T06:57:48.991+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:49.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:48 smithi155 bash[43742]: debug 2023-10-09T06:57:48.991+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:49.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.003+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:49.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.003+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:49.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.003+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:49.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.003+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:20.732312+0000 front 2023-10-09T06:57:20.732302+0000 (oldest deadline 2023-10-09T06:57:45.432137+0000) 2023-10-09T06:57:50.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[43742]: debug 2023-10-09T06:57:49.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:50.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[43742]: debug 2023-10-09T06:57:49.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:50.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[43742]: debug 2023-10-09T06:57:49.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:50.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[43742]: debug 2023-10-09T06:57:49.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:50.394 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:53.426092+0000 front 2023-10-09T06:56:53.425911+0000 (oldest deadline 2023-10-09T06:57:19.326057+0000) 2023-10-09T06:57:50.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:00.427966+0000 front 2023-10-09T06:57:00.428125+0000 (oldest deadline 2023-10-09T06:57:26.327794+0000) 2023-10-09T06:57:50.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:13.731103+0000 front 2023-10-09T06:57:13.731192+0000 (oldest deadline 2023-10-09T06:57:36.630869+0000) 2023-10-09T06:57:50.395 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:49 smithi155 bash[39663]: debug 2023-10-09T06:57:49.991+0000 7ff4ec466700 -1 osd.6 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:20.732312+0000 front 2023-10-09T06:57:20.732302+0000 (oldest deadline 2023-10-09T06:57:45.432137+0000) 2023-10-09T06:57:51.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:50 smithi155 bash[43742]: debug 2023-10-09T06:57:50.951+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:51.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:50 smithi155 bash[43742]: debug 2023-10-09T06:57:50.951+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:51.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:50 smithi155 bash[43742]: debug 2023-10-09T06:57:50.951+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:51.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:50 smithi155 bash[43742]: debug 2023-10-09T06:57:50.951+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:51.702 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[158978]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T06:57:51.702 INFO:journalctl@ceph.osd.6.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[159044]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-6 2023-10-09T06:57:52.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[43742]: debug 2023-10-09T06:57:51.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:52.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[43742]: debug 2023-10-09T06:57:51.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:52.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[43742]: debug 2023-10-09T06:57:51.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:52.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:51 smithi155 bash[43742]: debug 2023-10-09T06:57:51.907+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:53.143 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:52 smithi155 bash[43742]: debug 2023-10-09T06:57:52.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:53.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:52 smithi155 bash[43742]: debug 2023-10-09T06:57:52.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:53.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:52 smithi155 bash[43742]: debug 2023-10-09T06:57:52.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:53.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:52 smithi155 bash[43742]: debug 2023-10-09T06:57:52.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:53.144 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:52 smithi155 bash[43742]: debug 2023-10-09T06:57:52.947+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:53.899 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T06:57:53.976 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:57:53.977 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T06:57:53.977 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T06:57:53.977 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T06:57:54.167 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:53 smithi155 bash[43742]: debug 2023-10-09T06:57:53.955+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:54.167 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:53 smithi155 bash[43742]: debug 2023-10-09T06:57:53.955+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:54.167 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:53 smithi155 bash[43742]: debug 2023-10-09T06:57:53.955+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:54.168 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:53 smithi155 bash[43742]: debug 2023-10-09T06:57:53.955+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:54.168 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:53 smithi155 bash[43742]: debug 2023-10-09T06:57:53.955+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:54.168 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 systemd[1]: Stopping Ceph osd.7 for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:57:54.168 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.159+0000 7f9a73f37700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-09T06:57:54.643 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.159+0000 7f9a73f37700 -1 osd.7 406 *** Got signal Terminated *** 2023-10-09T06:57:54.644 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.159+0000 7f9a73f37700 -1 osd.7 406 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:57:55.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:55.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:55.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:55.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:55.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:54 smithi155 bash[43742]: debug 2023-10-09T06:57:54.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:56.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.015+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:56.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.015+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:56.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.015+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:56.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.015+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:56.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.015+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:57.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.971+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:57.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.971+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:57.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.971+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:57.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.971+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:57.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:56 smithi155 bash[43742]: debug 2023-10-09T06:57:56.971+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:58.393 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:57 smithi155 bash[43742]: debug 2023-10-09T06:57:57.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:58.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:57 smithi155 bash[43742]: debug 2023-10-09T06:57:57.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:58.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:57 smithi155 bash[43742]: debug 2023-10-09T06:57:57.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:58.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:57 smithi155 bash[43742]: debug 2023-10-09T06:57:57.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:58.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:57 smithi155 bash[43742]: debug 2023-10-09T06:57:57.975+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:57:59.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:58 smithi155 bash[43742]: debug 2023-10-09T06:57:58.983+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6806 osd.0 since back 2023-10-09T06:56:51.706707+0000 front 2023-10-09T06:56:51.706229+0000 (oldest deadline 2023-10-09T06:57:17.606245+0000) 2023-10-09T06:57:59.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:58 smithi155 bash[43742]: debug 2023-10-09T06:57:58.983+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6814 osd.1 since back 2023-10-09T06:57:05.109174+0000 front 2023-10-09T06:57:05.109488+0000 (oldest deadline 2023-10-09T06:57:26.809021+0000) 2023-10-09T06:57:59.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:58 smithi155 bash[43742]: debug 2023-10-09T06:57:58.983+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6822 osd.2 since back 2023-10-09T06:57:12.610815+0000 front 2023-10-09T06:57:12.610843+0000 (oldest deadline 2023-10-09T06:57:36.110748+0000) 2023-10-09T06:57:59.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:58 smithi155 bash[43742]: debug 2023-10-09T06:57:58.983+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.125:6830 osd.3 since back 2023-10-09T06:57:23.613598+0000 front 2023-10-09T06:57:23.613389+0000 (oldest deadline 2023-10-09T06:57:44.713065+0000) 2023-10-09T06:57:59.394 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:57:58 smithi155 bash[43742]: debug 2023-10-09T06:57:58.983+0000 7f9a6ceac700 -1 osd.7 406 heartbeat_check: no reply from 172.21.15.155:6804 osd.4 since back 2023-10-09T06:57:27.614385+0000 front 2023-10-09T06:57:27.614442+0000 (oldest deadline 2023-10-09T06:57:52.914288+0000) 2023-10-09T06:58:00.319 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:58:00 smithi155 bash[159187]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T06:58:00.319 INFO:journalctl@ceph.osd.7.smithi155.stdout:Oct 09 06:58:00 smithi155 bash[159254]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-osd-7 2023-10-09T06:58:01.886 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:01 smithi125 bash[72578]: level=warn ts=2023-10-09T06:58:01.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi125.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.125:8443: connect: connection refused" 2023-10-09T06:58:01.887 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:01 smithi125 bash[72578]: level=warn ts=2023-10-09T06:58:01.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi155.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.155:8443: connect: connection refused" 2023-10-09T06:58:02.308 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T06:58:02.337 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:58:02.337 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T06:58:02.338 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T06:58:02.338 DEBUG:teuthology.orchestra.run.smithi155:> sudo systemctl stop ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T06:58:02.580 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 systemd[1]: Stopping Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:58:02.893 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.578Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:58:02.894 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.579Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-09T06:58:02.895 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.579Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:58:02.895 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.583Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:58:02.895 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.583Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-09T06:58:02.895 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:02 smithi155 bash[54956]: ts=2023-10-09T06:58:02.583Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-09T06:58:03.363 DEBUG:teuthology.orchestra.run.smithi155:> sudo pkill -f 'journalctl -f -n 0 -u ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T06:58:03.371 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:03 smithi155 bash[159395]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:58:03.372 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:03 smithi155 bash[159456]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-09T06:58:03.372 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:03 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:58:03.372 INFO:journalctl@ceph.prometheus.a.smithi155.stdout:Oct 09 06:58:03 smithi155 systemd[1]: Stopped Ceph prometheus.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:58:03.390 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:58:03.390 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T06:58:03.391 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:58:10.313 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:09 smithi125 systemd[1]: Stopping Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:58:11.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:10 smithi125 bash[265112]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T06:58:11.136 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:10 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:58:11.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:10 smithi125 bash[265177]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-09T06:58:11.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:10 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:58:11.137 INFO:journalctl@ceph.node-exporter.a.smithi125.stdout:Oct 09 06:58:10 smithi125 systemd[1]: Stopped Ceph node-exporter.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:58:11.521 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:11 smithi125 systemd[1]: Stopping Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:58:11.521 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:11 smithi125 bash[72578]: level=info ts=2023-10-09T06:58:11.283Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:58:12.351 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:11 smithi125 bash[265237]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:58:12.351 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:12 smithi125 bash[265301]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-09T06:58:12.351 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:12 smithi125 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:58:12.351 INFO:journalctl@ceph.alertmanager.a.smithi125.stdout:Oct 09 06:58:12 smithi125 systemd[1]: Stopped Ceph alertmanager.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:58:20.252 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:58:26.854 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:26 smithi155 systemd[1]: Stopping Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:58:27.643 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:27 smithi155 bash[159713]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T06:58:27.644 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:27 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:58:27.644 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:27 smithi155 bash[159776]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-09T06:58:27.644 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:27 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:58:27.644 INFO:journalctl@ceph.node-exporter.b.smithi155.stdout:Oct 09 06:58:27 smithi155 systemd[1]: Stopped Ceph node-exporter.b for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:58:28.144 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:27 smithi155 systemd[1]: Stopping Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:58:28.144 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:27 smithi155 bash[53577]: t=2023-10-09T06:58:27+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-09T06:58:28.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:28 smithi155 bash[159839]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T06:58:28.893 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:28 smithi155 bash[159902]: Error response from daemon: No such container: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7-grafana-a 2023-10-09T06:58:28.894 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:28 smithi155 systemd[1]: ceph-95d0291c-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-09T06:58:28.894 INFO:journalctl@ceph.grafana.a.smithi155.stdout:Oct 09 06:58:28 smithi155 systemd[1]: Stopped Ceph grafana.a for 95d0291c-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:58:32.497 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:58:32.515 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:58:32.530 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T06:58:32.532 DEBUG:teuthology.misc:Transferring archived files from smithi125:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi125/crash 2023-10-09T06:58:32.533 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:58:32.591 INFO:teuthology.orchestra.run.smithi125.stderr:tar: /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:58:32.591 INFO:teuthology.orchestra.run.smithi125.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:58:32.593 DEBUG:teuthology.misc:Transferring archived files from smithi155:/var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi155/crash 2023-10-09T06:58:32.594 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:58:32.627 INFO:teuthology.orchestra.run.smithi155.stderr:tar: /var/lib/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:58:32.627 INFO:teuthology.orchestra.run.smithi155.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:58:32.629 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T06:58:32.629 DEBUG:teuthology.orchestra.run.smithi125:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T06:58:32.652 INFO:teuthology.orchestra.run.smithi125.stderr:grep: /var/log/ceph/95d0291c-6667-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T06:58:32.654 INFO:tasks.cephadm:Compressing logs... 2023-10-09T06:58:32.654 DEBUG:teuthology.orchestra.run.smithi125:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:58:32.698 DEBUG:teuthology.orchestra.run.smithi155:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:58:32.711 INFO:teuthology.orchestra.run.smithi125.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T06:58:32.714 INFO:teuthology.orchestra.run.smithi155.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-09T07:04:54.039 INFO:tasks.cephadm:Archiving logs... 2023-10-09T07:04:54.040 DEBUG:teuthology.misc:Transferring archived files from smithi125:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi125/log 2023-10-09T07:04:54.042 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T07:05:39.407 DEBUG:teuthology.misc:Transferring archived files from smithi155:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi155/log 2023-10-09T07:05:39.408 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T07:06:16.563 INFO:tasks.cephadm:Removing cluster... 2023-10-09T07:06:16.564 DEBUG:teuthology.orchestra.run.smithi125:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 --force 2023-10-09T07:06:18.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 95d0291c-6667-11ee-8db5-212e2dc638e7 --force 2023-10-09T07:06:20.362 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T07:06:20.362 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T07:06:20.370 DEBUG:teuthology.orchestra.run.smithi155:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T07:06:20.377 INFO:tasks.cephadm:Teardown complete 2023-10-09T07:06:20.377 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T07:06:20.388 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T07:06:20.389 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T07:06:20.413 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T07:06:20.495 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T07:06:20.504 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T07:06:20.505 DEBUG:teuthology.orchestra.run.smithi125:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:06:20.508 DEBUG:teuthology.orchestra.run.smithi155:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:06:20.552 INFO:teuthology.orchestra.run.smithi125.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T07:06:20.553 INFO:teuthology.orchestra.run.smithi125.stdout:============================================================================== 2023-10-09T07:06:20.553 INFO:teuthology.orchestra.run.smithi125.stdout:*hv01.front.sepi 44.4.53.2 2 u 122 128 377 0.162 3.753 0.905 2023-10-09T07:06:20.553 INFO:teuthology.orchestra.run.smithi125.stdout:+hv02.front.sepi 63.231.80.2 3 u 66 128 377 0.150 2.388 0.663 2023-10-09T07:06:20.553 INFO:teuthology.orchestra.run.smithi125.stdout:+hv03.front.sepi 74.6.168.72 3 u 15 128 377 0.170 2.450 0.868 2023-10-09T07:06:20.553 INFO:teuthology.orchestra.run.smithi125.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T07:06:20.559 INFO:teuthology.orchestra.run.smithi155.stdout: remote refid st t when poll reach delay offset jitter 2023-10-09T07:06:20.559 INFO:teuthology.orchestra.run.smithi155.stdout:============================================================================== 2023-10-09T07:06:20.559 INFO:teuthology.orchestra.run.smithi155.stdout:*hv01.front.sepi 44.4.53.2 2 u 67 128 377 0.206 2.592 0.647 2023-10-09T07:06:20.560 INFO:teuthology.orchestra.run.smithi155.stdout:+hv02.front.sepi 63.231.80.2 3 u 2 128 377 0.161 1.073 0.354 2023-10-09T07:06:20.560 INFO:teuthology.orchestra.run.smithi155.stdout:+hv03.front.sepi 74.6.168.72 3 u 62 128 377 0.169 0.915 0.401 2023-10-09T07:06:20.560 INFO:teuthology.orchestra.run.smithi155.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-09T07:06:20.561 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T07:06:20.571 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T07:06:20.572 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T07:06:20.581 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T07:06:20.589 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T07:06:20.599 INFO:teuthology.task.internal:Duration was 5088.107478 seconds 2023-10-09T07:06:20.599 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T07:06:20.608 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T07:06:20.608 DEBUG:teuthology.orchestra.run.smithi125:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T07:06:20.611 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T07:06:20.718 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T07:06:20.719 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi125.front.sepia.ceph.com 2023-10-09T07:06:20.719 DEBUG:teuthology.orchestra.run.smithi125:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T07:06:20.735 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi155.front.sepia.ceph.com 2023-10-09T07:06:20.735 DEBUG:teuthology.orchestra.run.smithi155:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T07:06:20.751 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T07:06:20.752 DEBUG:teuthology.orchestra.run.smithi125:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T07:06:20.781 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T07:06:20.954 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T07:06:20.955 DEBUG:teuthology.orchestra.run.smithi125:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T07:06:20.958 DEBUG:teuthology.orchestra.run.smithi155:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T07:06:21.429 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T07:06:21.441 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T07:06:21.441 DEBUG:teuthology.orchestra.run.smithi125:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T07:06:21.479 DEBUG:teuthology.orchestra.run.smithi155:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T07:06:21.516 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T07:06:21.527 DEBUG:teuthology.orchestra.run.smithi125:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:06:21.530 DEBUG:teuthology.orchestra.run.smithi155:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:06:21.555 INFO:teuthology.orchestra.run.smithi125.stdout:kernel.core_pattern = core 2023-10-09T07:06:21.572 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = core 2023-10-09T07:06:21.601 DEBUG:teuthology.orchestra.run.smithi125:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:06:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:06:21.636 DEBUG:teuthology.orchestra.run.smithi155:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:06:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:06:21.655 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T07:06:21.666 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T07:06:21.668 DEBUG:teuthology.misc:Transferring archived files from smithi125:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi125 2023-10-09T07:06:21.668 DEBUG:teuthology.orchestra.run.smithi125:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T07:06:21.736 DEBUG:teuthology.misc:Transferring archived files from smithi155:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516/remote/smithi155 2023-10-09T07:06:21.736 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T07:06:21.775 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T07:06:21.775 DEBUG:teuthology.orchestra.run.smithi125:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T07:06:21.777 DEBUG:teuthology.orchestra.run.smithi155:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T07:06:21.821 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T07:06:21.832 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T07:06:21.833 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T07:06:21.972 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T07:06:21.972 DEBUG:teuthology.orchestra.run.smithi125:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T07:06:21.974 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T07:06:21.980 INFO:teuthology.orchestra.run.smithi125.stdout: 1048617 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 07:06 /home/ubuntu/cephtest 2023-10-09T07:06:21.981 INFO:teuthology.orchestra.run.smithi125.stdout: 1311232 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 06:50 /home/ubuntu/cephtest/mnt.0 2023-10-09T07:06:21.981 INFO:teuthology.orchestra.run.smithi125.stdout: 1311233 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 06:50 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T07:06:21.982 INFO:teuthology.orchestra.run.smithi125.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T07:06:21.983 INFO:teuthology.orchestra.run.smithi155.stdout: 1048618 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 07:06 /home/ubuntu/cephtest 2023-10-09T07:06:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:06:21.984 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi125 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T07:06:21.994 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T07:06:22.006 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T07:06:22.017 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi125 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T07:06:22.017 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T07:06:22.047 DEBUG:teuthology.nuke:shortname: smithi125 2023-10-09T07:06:22.048 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:06:22.069 DEBUG:teuthology.nuke:shortname: smithi155 2023-10-09T07:06:22.069 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:06:22.072 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi125.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:32:10.446807', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T07:06:22.094 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7417516', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:32:10.448279', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-09T07:06:22.121 INFO:teuthology.orchestra.console.smithi125:Power off 2023-10-09T07:06:22.121 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T07:06:22.145 DEBUG:teuthology.orchestra.console.smithi125:power off output: Chassis Power Control: Down/Off 2023-10-09T07:06:22.146 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:22.163 INFO:teuthology.orchestra.console.smithi155:Power off 2023-10-09T07:06:22.164 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T07:06:22.174 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T07:06:22.188 DEBUG:teuthology.orchestra.console.smithi155:power off output: Chassis Power Control: Down/Off 2023-10-09T07:06:22.188 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:22.214 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2023-10-09T07:06:26.175 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:26.215 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:26.287 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T07:06:26.326 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2023-10-09T07:06:30.290 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:30.327 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:30.403 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is on 2023-10-09T07:06:30.439 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2023-10-09T07:06:34.406 DEBUG:teuthology.orchestra.console.smithi125:pexpect command: ipmitool -H smithi125.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:34.440 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:06:34.520 DEBUG:teuthology.orchestra.console.smithi125:check power output: Chassis Power is off 2023-10-09T07:06:34.551 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is off 2023-10-09T07:06:34.622 INFO:teuthology.orchestra.console.smithi125:Power off completed 2023-10-09T07:06:34.652 INFO:teuthology.orchestra.console.smithi155:Power off completed 2023-10-09T07:06:34.757 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5088.1074776649475 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=a433327857fe46cc922ffa60c80bdf7d status: fail success: false 2023-10-09T07:06:34.757 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T07:06:34.832 INFO:teuthology.run:FAIL